Flow Definition File

Overview

The default flow definition file generated by the Gradle task hubCreateFlow includes flow settings, as well as example steps.

You must customize the example steps before running the flow. You can delete the steps you don't need, and you can duplicate the steps if you need multiple steps of the same type. However, you must assign a unique sequence number for each step.

To learn about flows, see About Flows.

To learn about step types, see About Steps.

Flow Settings

   {
    "name" : "your-flow-name",
    "description" : "This is the default flow containing all of the default steps, generated by hubCreateFlow, plus additional settings and examples.",
    "batchSize" : 100,
    "threadCount" : 4,
    "options" : {
      "sourceQuery" : null,
      "provenanceGranularityLevel" : "fine"
    },
    "steps" : {
      "1" : { ... },
      "2" : { ... },
      "3" : { ... },
      "4" : { ... }
    }
  }

Ingestion Step Settings

   "1" : {
    "name" : "ingestion-step",
    "description" : "This is my ingestion step.",
    "stepDefinitionName" : "default-ingestion",
    "stepDefinitionType" : "INGESTION",
    "batchSize" : 100,
    "threadCount" : 4,
    "customHook" : {
      "module" : "/custom-modules/your-step-type/your-hook-directory/your-hook-module-name.sjs",
      "parameters" : {},
      "user" : "flow-operator",
      "runBefore" : false
    },
    "options" : {
      "targetDatabase" : "data-hub-STAGING",
      "sourceQuery" : null,
      "outputFormat" : "json",
      "collections" : [ "default-ingestion" ]
    },
    "fileLocations" : {
      "inputFilePath" : "path/to/folder",
      "inputFileType" : "json",
      "outputURIReplacement" : "output/URI,'substitute/URI'"
    }
  },

Mapping Step Settings

   "2" : {
    "name" : "mapping-step",
    "description" : "This is my mapping step.",
    "stepDefinitionName" : "default-mapping",
    "stepDefinitionType" : "MAPPING",
    "batchSize" : 100,
    "threadCount" : 4,
    "customHook" : {
      "module" : "/custom-modules/your-step-type/your-hook-directory/your-hook-module-name.sjs",
      "parameters" : {},
      "user" : "flow-operator",
      "runBefore" : false
    },
    "options" : {
      "sourceDatabase" : "data-hub-STAGING",
      "targetDatabase" : "data-hub-FINAL",
      "sourceQuery" : "cts.collectionQuery('default-ingestion')",
      "collections" : [ "default-mapping" ],
      "targetEntity" : "entity-name",
      "provenanceGranularityLevel" : "fine",
      "mapping" : {
        "name" : "mapping-name",
        "version" : 1
      }
    }
  },

Mastering Step Settings

   "3" : {
    "name" : "mastering-step",
    "description" : "This is my mastering step.",
    "stepDefinitionName" : "default-mastering",
    "stepDefinitionType" : "MASTERING",
    "batchSize" : 100,
    "threadCount" : 4,
    "customHook" : {
      "module" : "/custom-modules/your-step-type/your-hook-directory/your-hook-module-name.sjs",
      "parameters" : {},
      "user" : "flow-operator",
      "runBefore" : false
    },
    "options" : {
      "sourceDatabase" : "data-hub-FINAL",
      "targetDatabase" : "data-hub-FINAL",
      "sourceQuery" : "cts.andQuery([cts.collectionQuery('default-mapping'),cts.collectionQuery('mdm-content')])",
      "collections" : [ "default-mastering, mastered" ],
      "targetEntity" : "entity-name",
      "provenanceGranularityLevel" : "fine",
      "matchOptions" : { ... },
      "mergeOptions" : { ... }
    }
  },

   "matchOptions" : {
    "dataFormat" : "json",
    "propertyDefs" : {
      "property" : [
        {
          "name" : "ssn",
          "namespace" : "",
          "localname" : "IdentificationID"
        },
      ]
    },
    "algorithms" : {
      "algorithm" : [
        {
          "name" : "std-reduce",
          "function" : "standard-reduction",
          "namespace" : "",
          "at" : ""
        },
      ]
    },
    "collections" : {
      "content" : [ "my-content-collection" ]
    },
    "scoring" : {
      "add" : [
        {
          "propertyName" : "ssn",
          "weight" : "50"
        },
      ],
      "expand" : [
        {
          "propertyName" : "first-name",
          "algorithmRef" : "thesaurus",
          "weight" : "6",
          "thesaurus" : "/mdm/config/thesauri/first-name-synonyms.xml"
        },
        {
          "propertyName" : "last-name",
          "algorithmRef" : "dbl-metaphone",
          "weight" : "8",
          "dictionary" : "name-dictionary.xml",
          "distanceThreshold" : "50"
        }
      ],
      "reduce" : [
        {
          "algorithmRef" : "std-reduce",
          "weight" : "4",
          "allMatch" : { "property" : ["last-name", "addr1"] }
        }
      ]
    },
    "actions" : {
      "action" : [
        {
          "name" : "my-custom-action",
          "function" : "custom-action",
          "namespace" : "http://marklogic.com/smart-mastering/action",
          "at" : "/custom-action.xqy"
        }
      ]
    },
    "thresholds" : {
      "threshold" : [
        { "above" : "30", "label" : "Possible Match" },
        { "above" : "50", "label" : "Likely Match", "action" : "notify" },
        { "above" : "68", "label" : "Definitive Match", "action" : "merge" },
        { "above" : "75", "label" : "Custom Match", "action" : "my-custom-action" }
      ]
    },
    "tuning" : {
      "maxScan" : 200
    }
  },

   "mergeOptions" : {
    "matchOptions" : "mlw-match",
    "propertyDefs" : {
      "properties" : [
        {
          "name" : "ssn",
          "localname" : "IdentificationID",
          "namespace" : ""
        },
        {
          "name" : "shallow",
          "path" : "/es:envelope/es:headers/shallow"
        }
      ],
      "namespaces" : {
        "has" : "has",
        "m" : "http://marklogic.com/smart-mastering/merging",
        "es" : "http://marklogic.com/entity-services"
      }
    },
    "algorithms" : {
      "stdAlgorithm" : {
        "timestamp" : { "path" : "/es:envelope/es:headers/sm:sources/sm:source/sm:dateTime" },
        "namespaces" : {
          "sm" : "http://marklogic.com/smart-mastering",
          "es" : "http://marklogic.com/entity-services"
        }
      },
      "custom" : [
        {
          "name" : "customMerge",
          "function" : "doCustomMerge",
          "namespace" : "http://marklogic.com/smart-mastering/merging",
          "at" : "/custom-merge-xqy.xqy"
        }
      ],
      "collections" : {
        "onMerge" : {
          "function" : "collections",
          "namespace" : "test/merge-collection-algorithm",
          "at" : "/test/suites/customizing-collections/lib/merged-collections.xqy"
        },
        "onArchive" : {
          "remove" : { "collection" : ["Entity"] },
          "add" : { "collection" : ["custom-archived"] }
        },
        "onNoMatch" : {
          "function" : "noMatchCollections",
          "namespace" : "",
          "at" : "/test/suites/customizing-collections/lib/noMatchCollections.sjs"
        },
        "onNotification" : {
          "set" : { "collection" : ["notification"] }
        }
      }
    },
    "mergeStrategies" : [
      {
        "name" : "crm-source-weight",
        "algorithmRef" : "standard",
        "sourceWeights" : [
          {
            "source" : {
              "name" : "CRM",
              "weight" : "10"
            }
          }
        ]
      },
      {
        "name" : "length-weight",
        "algorithmRef" : "standard",
        "maxValues" : "1",
        "length" : { "weight" : "10" }
      }
    ],
    "merging" : [
      {
        "propertyName" : "ssn",
        "maxValues" : "1",
        "maxSources" : "1",
        "strategy" : "crm-source-weight"
      },
      {
        "propertyName" : "name",
        "maxValues" : "1",
        "doubleMetaphone" : {
          "distanceThreshold" : "50"
        },
        "synonymsSupport" : "true",
        "thesaurus" : "/mdm/config/thesauri/first-name-synonyms.xml",
        "length" : { "weight" : "8" }
      },
      {
        "propertyName" : "dob",
        "maxValues" : "1",
        "algorithmRef" : "standard",
        "sourceWeights" : {
          "source" : {
            "name" : "better-source",
            "weight" : "4"
          }
        }
      },
      {
        "default" : "true",
        "strategy" : "crm-source-weight"
      }
    ],
    "tripleMerge" : {
      "function" : "custom-trips",
      "namespace" : "http://marklogic.com/smart-mastering/merging",
      "at" : "/custom-triple-merge.xqy",
      "some-param" : 3
    }
  }

Custom Step Settings

   "4" : {
    "name" : "custom-step",
    "description" : "",
    "stepDefinitionName" : "custom-step-def",
    "stepDefinitionType" : "CUSTOM",
    "retryLimit" : null
    "batchSize" : 100,
    "threadCount" : 4,
    "customHook" : {
      "module" : "/custom-modules/your-step-type/your-hook-directory/your-hook-module-name.sjs",
      "parameters" : {},
      "user" : "flow-operator",
      "runBefore" : false
    },
    "options" : {
      "sourceDatabase" : "data-hub-STAGING",
      "targetDatabase" : "data-hub-FINAL",
      "sourceQuery" : "cts.collectionQuery('my-custom-query')",
      "outputFormat" : "json",
      "collections" : [ "my-collection-tag" ],
      "provenanceGranularityLevel" : "fine",
    }
  }

Full Flow Definition

The default flow definition file created by Gradle contains the most common settings. The following flow definition includes additional settings that are also allowed in the flow definition.

Hover over a line in the code below to view information about that property, if available. Click to make the info box sticky; click again to unstick.

   {
    "name" : "your-flow-name",
The human-friendly name of the flow.
    "description" : "This is the default flow containing all of the default steps, generated by hubCreateFlow, plus additional settings and examples.",
(Optional) A description of the flow.
    "batchSize" : 100,
The number of documents to process per batch.
    "threadCount" : 4,
The number of threads to use when running a flow.
    "options" : {
Key-value pairs to pass as parameters to custom modules in every step in the flow.
      "sourceQuery" : null,
The CTS query to use to select the source data to process. To filter by a collection tag, use cts.collectionQuery('my-collection-name'). Example: "sourceQuery" : "cts.collectionQuery('default-ingestion')" See CTS Query.
      "provenanceGranularityLevel" : "fine"
The granularity of the provenance tracking information: coarse (default) to store document-level provenance information only or fine to store document-level and property-level provenance information. Applies only to mapping, mastering, and custom steps.
    },
    "steps" : {
The steps to be run within the flow. Each step in the flow has a sequence number and a customized copy of the step definition. The step definition differs based on the step type (ingestion, mapping, mastering, or custom).
      "1" : {
A string containing a number which represents the order of the step in the sequence. Note: The steps can be listed in any order, as long as the keys are unique within the steps node of the flow. Duplicate keys can produced unexpected results. The key number must be greater than 0.
        "name" : "ingestion-step",
The name of the step instance.
        "description" : "This is my ingestion step.",
A description of the step.
        "stepDefinitionName" : "default-ingestion",
The name of a step definition, which could be one of the default step definitions or a custom one. Custom step definitions can be created using QuickStart or using the Gradle task hubCreateStepDefinition. Tip: If you are customizing a default step type (ingestion, mapping, or mastering), leave the value as default-ingestion, default-mapping, or default-mastering.
        "stepDefinitionType" : "INGESTION",
The type of the step definition: INGESTION, MAPPING, MASTERING, or CUSTOM.
        "batchSize" : 100,
The number of documents to process per batch. If not defined or if set to 0 or null in the step settings, the value in the flow settings is used.
        "threadCount" : 4,
The number of threads to use when running a flow. If not defined or if set to 0 or null in the step settings, the value in the flow settings is used.
        "customHook" : {
Definition of a hook that performs additional processes before or after the step. See Creating a Custom Hook Module and Adding a Custom Hook to a Step Manually.
          "module" : "/custom-modules/your-step-type/your-hook-directory/your-hook-module-name.sjs",
The path to your custom hook module.
          "parameters" : {},
Parameters, as key-value pairs, to pass to your custom hook module.
          "user" : "flow-operator",
The user account to use to run the module. Default is the user running the flow; e.g., flow-operator.
          "runBefore" : false
For a pre-step hook, set to true. For a post-step hook, set to false.
        },
        "options" : {
          "targetDatabase" : "data-hub-STAGING",
For ingestion, choose the STAGING database where you want to store the ingested data. Default is data-hub-STAGING.
          "sourceQuery" : null,
The CTS query to use to select the source data to process. To filter by a collection tag, use cts.collectionQuery('my-collection-name'). Example: "sourceQuery" : "cts.collectionQuery('default-ingestion')" See CTS Query.
          "outputFormat" : "json",
The format of the processed record: Text, JSON, XML, or Binary.
          "collections" : [ "default-ingestion" ]
The collection tags to assign to the resulting records.
        },
        "fileLocations" : {
          "inputFilePath" : "path/to/folder",
The location of your source files.
          "inputFileType" : "json",
The format of your source files: Text, JSON, XML, Binary, or Delimited Text.
          "outputURIReplacement" : "output/URI,'substitute/URI'"
A comma-separated list of replacements used to customize the URIs of the ingested records. The list is comprised of regular expression patterns and their replacement strings in the format pattern,'string',pattern,'string',.... The replacement strings must be enclosed in single quotes.
        }
      },
      "2" : {
A string containing a number which represents the order of the step in the sequence. Note: The steps can be listed in any order, as long as the keys are unique within the steps node of the flow. Duplicate keys can produced unexpected results. The key number must be greater than 0.
        "name" : "mapping-step",
The name of the step instance.
        "description" : "This is my mapping step.",
A description of the step.
        "stepDefinitionName" : "default-mapping",
The name of a step definition, which could be one of the default step definitions or a custom one. Custom step definitions can be created using QuickStart or using the Gradle task hubCreateStepDefinition. Tip: If you are customizing a default step type (ingestion, mapping, or mastering), leave the value as default-ingestion, default-mapping, or default-mastering.
        "stepDefinitionType" : "MAPPING",
The type of the step definition: INGESTION, MAPPING, MASTERING, or CUSTOM.
        "batchSize" : 100,
The number of documents to process per batch. If not defined or if set to 0 or null in the step settings, the value in the flow settings is used.
        "threadCount" : 4,
The number of threads to use when running a flow. If not defined or if set to 0 or null in the step settings, the value in the flow settings is used.
        "customHook" : {
Definition of a hook that performs additional processes before or after the step. See Creating a Custom Hook Module and Adding a Custom Hook to a Step Manually.
          "module" : "/custom-modules/your-step-type/your-hook-directory/your-hook-module-name.sjs",
The path to your custom hook module.
          "parameters" : {},
Parameters, as key-value pairs, to pass to your custom hook module.
          "user" : "flow-operator",
The user account to use to run the module. Default is the user running the flow; e.g., flow-operator.
          "runBefore" : false
For a pre-step hook, set to true. For a post-step hook, set to false.
        },
        "options" : {
          "sourceDatabase" : "data-hub-STAGING",
For mapping, choose the STAGING database where you obtained the data. Default is data-hub-STAGING.
          "targetDatabase" : "data-hub-FINAL",
For mapping, choose the FINAL database where you want to store mapped data. Default is data-hub-FINAL.
          "sourceQuery" : "cts.collectionQuery('default-ingestion')",
The CTS query to use to select the source data to process. To filter by a collection tag, use cts.collectionQuery('my-collection-name'). Example: "sourceQuery" : "cts.collectionQuery('default-ingestion')" See CTS Query.
          "collections" : [ "default-mapping", "mdm-content" ],
The collection tags to assign to the resulting records.
          "targetEntity" : "entity-name",
The entity to map against the source data.
          "provenanceGranularityLevel" : "fine",
The granularity of the provenance tracking information: coarse (default) to store document-level provenance information only or fine to store document-level and property-level provenance information.
          "mapping" : {
How to map the properties of the targetEntity to the fields of the source data.
            "name" : "mapping-name",
The name of your mapping that is defined in your-project-root/mappings/your-mapping-name/mapping.version.json.
            "version" : 1
The version of the mapping to use. Your mapping must be defined in your-project-root/mappings/your-mapping-name/mapping.version.json.
          }
        }
      },
      "3" : {
A string containing a number which represents the order of the step in the sequence. Note: The steps can be listed in any order, as long as the keys are unique within the steps node of the flow. Duplicate keys can produced unexpected results. The key number must be greater than 0.
        "name" : "mastering-step",
The name of the step instance.
        "description" : "This is my mastering step.",
A description of the step.
        "stepDefinitionName" : "default-mastering",
The name of a step definition, which could be one of the default step definitions or a custom one. Custom step definitions can be created using QuickStart or using the Gradle task hubCreateStepDefinition. Tip: If you are customizing a default step type (ingestion, mapping, or mastering), leave the value as default-ingestion, default-mapping, or default-mastering.
        "stepDefinitionType" : "MASTERING",
The type of the step definition: INGESTION, MAPPING, MASTERING, or CUSTOM.
        "batchSize" : 100,
The number of documents to process per batch. If not defined or if set to 0 or null in the step settings, the value in the flow settings is used.
        "threadCount" : 4,
The number of threads to use when running a flow. If not defined or if set to 0 or null in the step settings, the value in the flow settings is used.
        "customHook" : {
Definition of a hook that performs additional processes before or after the step. See Creating a Custom Hook Module and Adding a Custom Hook to a Step Manually.
          "module" : "/custom-modules/your-step-type/your-hook-directory/your-hook-module-name.sjs",
The path to your custom hook module.
          "parameters" : {},
Parameters, as key-value pairs, to pass to your custom hook module.
          "user" : "flow-operator",
The user account to use to run the module. Default is the user running the flow; e.g., flow-operator.
          "runBefore" : false
For a pre-step hook, set to true. For a post-step hook, set to false.
        },
        "options" : {
          "sourceDatabase" : "data-hub-FINAL",
For mastering, choose the FINAL database where you stored processed data. Default is data-hub-FINAL.
          "targetDatabase" : "data-hub-FINAL",
For mastering, choose the FINAL database where you want to store mastered data. Default is data-hub-FINAL. Note: For a mastering step, the source database and the target database must be the same. If duplicates are found, the original records are archived and the merged version is added to the same database. If you want the target database to be different, you can create a custom step with a custom module to override the default behavior of the mastering step.
          "sourceQuery" : "cts.andQuery([cts.collectionQuery('default-mapping'),cts.collectionQuery('mdm-content')])",
The CTS query to use to select the source data to process. To filter by a collection tag, use cts.collectionQuery('my-collection-name'). Example: "sourceQuery" : "cts.collectionQuery('default-ingestion')" See CTS Query.
          "collections" : [ "default-mastering, mastered" ],
The collection tags to assign to the resulting records.
          "targetEntity" : "entity-name",
The entity to map against the source data.
          "provenanceGranularityLevel" : "fine",
The granularity of the provenance tracking information: coarse (default) to store document-level provenance information only or fine to store document-level and property-level provenance information.
          "matchOptions" : {
The settings used to find potential matches. See Smart Mastering Core - Matching Options.
            "dataFormat" : "json",
The format of your source records: Text, JSON, XML, or Binary.
            "propertyDefs" : {
Definitions of properties to compare.
              "property" : [
                {
                  "name" : "ssn",
The alias for this property definition.
                  "namespace" : "",
(Optional) The namespace that encompasses the XML element or JSON property (record field) to compare.
                  "localname" : "IdentificationID"
The name of the XML element or JSON property (record field) to compare.
                },
              ]
            },
            "algorithms" : {
Definitions of algorithms that compare values. Each algorithm corresponds to a match type (Exact, Synonym, Double Metaphone, Reduce, Zip, and Custom). The default algorithm is that of the Exact match type, which determines whether two values are equal.
              "algorithm" : [
                {
                  "name" : "std-reduce",
The alias for this algorithm definition.
                  "function" : "standard-reduction",
The function to run if this algorithm definition is selected.
                  "namespace" : "",
(Optional) The namespace of the module that contains the function.
                  "at" : ""
The path to the module that contains the function.
                },
              ]
            },
            "collections" : {
A set of collections that overrides the default collection used to determine the scope of the dataset being compared. If multiple content elements are specified, the dataset is restricted to an intersection of those collections.
              "content" : [ "my-content-collection" ]
One or more collections used to determine the dataset to be compared.
            },
            "scoring" : {
Rules (add, expand, reduce) that define how the comparison is scored based on assigned weights. The maximum possible score is the sum of the weights of all of the weight attributes. The match process uses the simple scoring option, with the property weight controlling how much influence each should have. See Relevance Scores.
              "add" : [
Properties whose values are simply compared between the records and, if the values match exactly, the assigned weight is added to the score.
                {
                  "propertyName" : "ssn",
The alias of a property definition under the matchOptions/propertyDefs node of this step.
                  "weight" : "50"
The weight added to the score if the property values of two records match exactly.
                },
              ],
              "expand" : [
Properties whose values are compared using a different algorithm to determine a match. For example, the property values can be considered a positive match if one is a synonym of the other or if both values phonetically sound alike. If so, the assigned weight is added to the score.
                {
                  "propertyName" : "first-name",
The alias of a property definition under the matchOptions/propertyDefs node of this step.
                  "algorithmRef" : "thesaurus",
The alias of an algorithm definition under the matchOptions/algorithms node of this step.
                  "weight" : "6",
The weight added to the score if the property values of two records are considered a match based on the selected algorithm.
                  "thesaurus" : "/mdm/config/thesauri/first-name-synonyms.xml"
The location of the thesaurus that is stored in a MarkLogic Server database and used to determine synonyms. See also: Managing Thesaurus Documents
                },
                {
                  "propertyName" : "last-name",
The alias of a property definition under the matchOptions/propertyDefs node of this step.
                  "algorithmRef" : "dbl-metaphone",
The alias of an algorithm definition under the matchOptions/algorithms node of this step.
                  "weight" : "8",
The weight added to the score if the property values of two records are considered a match based on the selected algorithm.
                  "dictionary" : "name-dictionary.xml",
The location of the phonetic dictionary that is stored in a database and used when comparing words phonetically. See also: Custom Dictionaries
                  "distanceThreshold" : "50"
The threshold below which the phonetic difference (distance) between two strings is considered insignificant; i.e., the strings are similar to each other.
                }
              ],
              "reduce" : [
Combinations of properties whose matching values might be a false match. For example, two members of the same family with the same last names and addresses might be misinterpreted as being the same person. In this case, the score is reduced by the assigned weight to give the match less importance.
                {
                  "algorithmRef" : "std-reduce",
The alias of an algorithm definition under the matchOptions/algorithms node of this step.
                  "weight" : "4",
A positive integer that denotes how much to reduce the weight of a match.
                  "allMatch" : { "property" : ["last-name", "addr1"] }
The combination of properties that might falsely indicate a match if the values of these properties are equal between two records.
                }
              ]
            },
            "actions" : {
Custom actions that can be performed when a threshold is reached. See Custom Match Actions.
              "action" : [
The custom action to perform when a threshold is reached.
                {
                  "name" : "my-custom-action",
The alias for this action definition.
                  "function" : "custom-action",
The function to run if this action definition is selected.
                  "namespace" : "http://marklogic.com/smart-mastering/action",
(Optional) The namespace of the module that contains the function.
                  "at" : "/custom-action.xqy"
The path to the module that contains the function.
                }
              ]
            },
            "thresholds" : {
Score thresholds that trigger an action.
              "threshold" : [
A score threshold definition, including the action to perform if the threshold is exceeded.
                { "above" : "30", "label" : "Possible Match" },
                { "above" : "50", "label" : "Likely Match", "action" : "notify" },
                { "above" : "68", "label" : "Definitive Match", "action" : "merge" },
                {
                  "above" : "75",
The score threshold. If the match score exceeds this value, the action is performed.
                  "label" : "Custom Match",
The alias for this threshold definition.
                  "action" : "my-custom-action"

    The action to perform if the score is above the threshold. Possible values:
    
notify creates a notification record in the FINAL database with information about the match.
merge creates a new record with the combined properties of the original records that match, then archives the old records.
The alias of an action definition under the matchOptions/actions node of this step.
  
                }
              ]
            },
            "tuning" : {
              "maxScan" : 200
The maximum number of highest scoring potential matches that will be considered for merging.
            }
          },
          "mergeOptions" : {
The settings used to merge records that match. See Smart Mastering Core - Merging Options.
            "matchOptions" : "mlw-match",
The name of a set of match options that were previously stored in the server. See Saving Options.
            "propertyDefs" : {
Definitions of properties to merge.
              "properties" : [
                {
                  "name" : "ssn",
The alias for this property definition.
                  "localname" : "IdentificationID",
The name of the XML element or JSON property (record field) to merge.
                  "namespace" : ""
(Optional) The namespace that encompasses the XML element or JSON property (record field) to merge.
                },
                {
                  "name" : "shallow",
The alias for this property definition.
                  "path" : "/es:envelope/es:headers/shallow"
Path leading to the headers or instance sections of records, where the merge properties are defined. 
XML:  /es:envelope/es:headers
JSON: /envelope/headers
XML:  /es:envelope/es:instance
JSON: /envelope/instance
 Note: Namespaces in the path must be defined in propertyDefs/namespaces node.
                }
              ],
              "namespaces" : {
Key-value pairs that assign aliases to namespaces. The key is the alias and the value is the full namespace.
                "has" : "has",
                "m" : "http://marklogic.com/smart-mastering/merging",
                "es" : "http://marklogic.com/entity-services"
              }
            },
            "algorithms" : {
Definitions of algorithms that merge values.
              "stdAlgorithm" : {
The standard algorithm that implements the default merge behavior.
                "timestamp" : { "path" : "/es:envelope/es:headers/sm:sources/sm:source/sm:dateTime" },
The path to a timestamp field within the record. This field is used to determine which values to include in the merged property, based on their recency, up to the maximum number specified in the Max Values field in Merge Options (Standard) or in Merge Strategies. Namespaces used in the path must be defined within the record.
                "namespaces" : {
(Optional) Key-value pairs that assign aliases to namespaces. The key is the alias and the value is the full namespace.
                  "sm" : "http://marklogic.com/smart-mastering",
                  "es" : "http://marklogic.com/entity-services"
                }
              },
              "custom" : [
Definitions of custom algorithms that merge values.
                {
                  "name" : "customMerge",
The alias for this custom algorithm definition.
                  "function" : "doCustomMerge",
The custom merge function to run.
                  "namespace" : "http://marklogic.com/smart-mastering/merging",
(Optional) The namespace of the module that contains the function.
                  "at" : "/custom-merge-xqy.xqy"
The path to the module that contains the function.
                }
              ],
              "collections" : {
Rules that specify how collection tags are managed when an event occurs.
                "onMerge" : {
How collection tags are applied to the new record that was created when matching records are merged. The default set of collection tags is comprised of:
    
the union of collection tags from the original records,
plus mdm-content,
plus mdm-merged.
  
                  "function" : "collections",
The function that manage collection tags if the event occurs.
                  "namespace" : "test/merge-collection-algorithm",
(Optional) The namespace of the module that contains the function.
                  "at" : "/test/suites/customizing-collections/lib/merged-collections.xqy"
The path to the module that contains the function.
                },
                "onArchive" : {
How collection tags are applied to the original records after their content have been merged into a new record. The default set of collection tags is comprised of:
    
the collection tags from the original record,
minus mdm-content,
plus mdm-archived.
  
                  "remove" : { "collection" : ["Entity"] },
One or more collection tags to remove from the default union of tags.
                  "add" : { "collection" : ["custom-archived"] }
One or more collection tags to add to the default union of tags.
                },
                "onNoMatch" : {
How collection tags are applied to records that were not merged because no matches were found or because the total matching scores did not exceed the defined thresholds. The default set of collection tags is comprised of:
    
the collection tags from the original record,
plus mdm-content.
  
                  "function" : "noMatchCollections",
The function that manage collection tags if the event occurs.
                  "namespace" : "",
(Optional) The namespace of the module that contains the function.
                  "at" : "/test/suites/customizing-collections/lib/noMatchCollections.sjs"
The path to the module that contains the function.
                },
                "onNotification" : {
How collection tags are applied to notification records. The default set of collection tags is comprised of mdm-notification only.
                  "set" : { "collection" : ["notification"] }
One or more collection tags to replace the default union of tags.
                }
              }
            },
            "mergeStrategies" : [
Predefined configurations for merging.
              {
                "name" : "crm-source-weight",
The name for the strategy.
                "algorithmRef" : "standard",
The alias of an algorithm definition under the mergeOptions/algorithms node of this step.
                "sourceWeights" : [
The list of data sources and the weights assigned to them. If the set of matching records come from more sources than maxSources, the source weights are used to determine which records are included in the merge. Example: If maxSources is set to 1, only records from the highest weighted source are included in the merge.
                  {
                    "source" : {
                      "name" : "CRM",
The name of the source, exactly as shown in the envelopes of the records under headers » sources.
                      "weight" : "10"
The weight used to decide the priority of a source when merging.
                    }
                  }
                ]
              },
              {
                "name" : "length-weight",
                "algorithmRef" : "standard",
                "maxValues" : "1",
The maximum number of values to allow in the merged property. Default is 99.
                "length" : { "weight" : "10" }
The weight assigned to the length of a string.
              }
            ],
            "merging" : [
Rules that specify how to merge records that match.
              {
                "propertyName" : "ssn",
The alias of a property definition under the mergeOptions/propertyDefs node of this step.
                "maxValues" : "1",
The maximum number of values to allow in the merged property. Default is 99.
                "maxSources" : "1",
The maximum number of data sources from which to get values to merge. For example, to copy values from a single source, set maxSources to 1.
                "strategy" : "crm-source-weight"
The alias of a strategy definition under the mergeOptions/mergeStrategies node of this step.
              },
              {
                "propertyName" : "name",
                "maxValues" : "1",
                "doubleMetaphone" : {
If this setting is present, the Double Metaphone algorithm is used to determine the values to merge.
                  "distanceThreshold" : "50"
The threshold below which the phonetic difference (distance) between two strings is considered insignificant; i.e., the strings are similar to each other.
                },
                "synonymsSupport" : "true",
If true, synonyms are included in the list of values to merge. Synonyms are determined using the specified thesaurus.
                "thesaurus" : "/mdm/config/thesauri/first-name-synonyms.xml",
The location of the thesaurus that is stored in a MarkLogic Server database and used to determine synonyms. See also: Managing Thesaurus Documents
                "length" : { "weight" : "8" }
The weight assigned to the length of a string.
              },
              {
                "propertyName" : "dob",
                "maxValues" : "1",
                "algorithmRef" : "standard",
The alias of an algorithm definition under the mergeOptions/algorithms node of this step.
                "sourceWeights" : {
The list of data sources and the weights assigned to them. If the set of matching records come from more sources than maxSources, the source weights are used to determine which records are included in the merge. Example: If maxSources is set to 1, only records from the highest weighted source are included in the merge.
                  "source" : {
                    "name" : "better-source",
The name of the source, exactly as shown in the envelopes of the records under headers » sources.
                    "weight" : "4"
The weight used to decide the priority of a source when merging.
                  }
                }
              },
              {
                "default" : "true",
If true, the specified strategy is the default. Important: If this setting is present, do not include a propertyName setting.
                "strategy" : "crm-source-weight"
The alias of a strategy definition under the mergeOptions/mergeStrategies node of this step.
              }
            ],
            "tripleMerge" : {
Definition of an algorithm that merges triples.
              "function" : "custom-trips",
The function that merges triples.
              "namespace" : "http://marklogic.com/smart-mastering/merging",
The namespace of the module that contains the function.
              "at" : "/custom-triple-merge.xqy",
The path to the module that contains the function.
              "some-param" : 3
Parameters, as key-value pairs, to pass to your triple merge function.
            }
          }
        }
      },
      "4" : {
A string containing a number which represents the order of the step in the sequence. Note: The steps can be listed in any order, as long as the keys are unique within the steps node of the flow. Duplicate keys can produced unexpected results. The key number must be greater than 0.
        "name" : "custom-step",
The name of the step instance.
        "description" : "",
A description of the step.
        "stepDefinitionName" : "custom-step-def",
The name of a step definition, which could be one of the default step definitions or a custom one. Custom step definitions can be created using QuickStart or using the Gradle task hubCreateStepDefinition. Tip: If you are customizing a default step type (ingestion, mapping, or mastering), leave the value as default-ingestion, default-mapping, or default-mastering.
        "stepDefinitionType" : "CUSTOM",
The type of the step definition: INGESTION, MAPPING, MASTERING, or CUSTOM.
        "retryLimit" : null
The maximum number of times to retry running the step if the previous run failed.
        "batchSize" : 100,
The number of documents to process per batch. If not defined or if set to 0 or null in the step settings, the value in the flow settings is used.
        "threadCount" : 4,
The number of threads to use when running a flow. If not defined or if set to 0 or null in the step settings, the value in the flow settings is used.
        "customHook" : {
Definition of a hook that performs additional processes before or after the step. See Creating a Custom Hook Module and Adding a Custom Hook to a Step Manually.
          "module" : "/custom-modules/your-step-type/your-hook-directory/your-hook-module-name.sjs",
The path to your custom hook module.
          "parameters" : {},
Parameters, as key-value pairs, to pass to your custom hook module.
          "user" : "flow-operator",
The user account to use to run the module. Default is the user running the flow; e.g., flow-operator.
          "runBefore" : false
For a pre-step hook, set to true. For a post-step hook, set to false.
        },
        "options" : {
          "sourceDatabase" : "data-hub-STAGING",
For mapping, choose the STAGING database where you obtained the data. Default is data-hub-STAGING.
          "targetDatabase" : "data-hub-FINAL",
For mapping, choose the FINAL database where you want to store mapped data. Default is data-hub-FINAL.
          "sourceQuery" : "cts.collectionQuery('my-custom-query')",
The CTS query to use to select the source data to process. To filter by a collection tag, use cts.collectionQuery('my-collection-name'). Example: "sourceQuery" : "cts.collectionQuery('default-ingestion')" See CTS Query.
          "outputFormat" : "json",
The format of the processed record: Text, JSON, XML, or Binary.
          "collections" : [ "my-collection-tag" ],
The collection tags to assign to the resulting records.
          "provenanceGranularityLevel" : "fine"
The granularity of the provenance tracking information: coarse (default) to store document-level provenance information only or fine to store document-level and property-level provenance information.
        }
      }
    }
  }