フロー定義ファイル

概要

GradleタスクhubCreateFlowによって生成されたデフォルトの定義ファイルには、フローの設定およびステップのサンプルが含まれています。

ステップのサンプルは、フローの実行前に必ずカスタマイズする必要があります。不要なステップは削除できます。また同種のステップが複数必要な場合は、重複して持つことができます。その場合、一意のシーケンス番号（順番を示すもの）を割り当てる必要があります。

フローの詳細は、フローについてを参照してください。

ステップのタイプの詳細は、ステップについてを参照してください。

フロー定義ファイルのコンポーネント

• フローの設定
• 読み込みステップの設定
• マッピングステップの設定
• マッチングステップの設定
• マージングステップの設定
• マスタリングステップの設定
• カスタムステップの設定

フローの設定

   {
    "name" : "MyFlow",
    "description" : "This flow contains examples of steps plus additional settings.",
    "batchSize" : 100,
    "threadCount" : 4,
    "stopOnError" : false,
    "options" : {
      "sourceQuery" : null,
      "provenanceGranularityLevel" : "fine"
    },
    "steps" : {
      "1" : { ...},
      "2" : { ...},
      "3" : { ...},
      "4" : { ...}
    }
  }

読み込みステップの設定

   "1" : {
    "name" : "MyIngestionStep",
    "description" : "This is my ingestion step.",
    "stepDefinitionName" : "default-ingestion",
    "stepDefinitionType" : "INGESTION",
    "batchSize" : 100,
    "threadCount" : 4,
    "customHook" : {
      "module" : "/custom-modules/your-step-type/your-hook-directory/your-hook-module-name.sjs",
      "parameters" : {},
      "user" : "flow-operator",
      "runBefore" : false
    },
    "options" : {
      "stepUpdate" : false,
      "acceptsBatch" : false,
      "targetDatabase" : "data-hub-STAGING",
      "collections" : [ "default-ingestion" ],
      "additionalCollections" : [],
      "outputFormat" : "json"
    },
    "fileLocations" : {
      "inputFilePath" : "path/to/folder",
      "inputFileType" : "json",
      "outputURIReplacement" : "output/URI,'substitute/URI'"
    }
  },

マッピングステップの設定

   "2" : {
    "name" : "MyMappingStep",
    "description" : "This is my mapping step.",
    "stepDefinitionName" : "default-mapping",
    "stepDefinitionType" : "MAPPING",
    "batchSize" : 100,
    "threadCount" : 4,
    "customHook" : {
      "module" : "/custom-modules/your-step-type/your-hook-directory/your-hook-module-name.sjs",
      "parameters" : {},
      "user" : "flow-operator",
      "runBefore" : false
    },
    "options" : {
      "stepUpdate" : false,
      "acceptsBatch" : false,
      "sourceDatabase" : "data-hub-STAGING",
      "sourceCollection" : "MyIngestionStep",
      "sourceQuery" : "cts.collectionQuery('my-custom-query')",
      "constrainSourceQueryToJob" : false,
      "targetEntity" : "MyEntity",
      "validateEntity" : false,
      "targetDatabase" : "data-hub-FINAL",
      "collections" : [ "default-mapping" ],
      "additionalCollections" : [],
      "outputFormat" : "json",
      "provenanceGranularityLevel" : "fine",
      "mapping" : {
        "name" : "mapping-name",
        "version" : "1"
      }
    }
  },

マッチングステップの設定

   "3" : {
    "name" : "MyMatchingStep",
    "description" : "This is my matching step.",
    "stepDefinitionName" : "default-matching",
    "stepDefinitionType" : "MATCHING",
    "batchSize" : 100,
    "threadCount" : 4,
    "customHook" : {
      "module" : "/custom-modules/your-step-type/your-hook-directory/your-hook-module-name.sjs",
      "parameters" : {},
      "user" : "flow-operator",
      "runBefore" : false
    },
    "options" : {
      "stepUpdate" : false,
      "acceptsBatch" : false,
      "sourceDatabase" : "data-hub-FINAL",
      "sourceCollection" : "MyMappingStep",
      "sourceQuery" : "cts.collectionQuery('my-custom-query')",
      "constrainSourceQueryToJob" : false,
      "targetEntity" : "MyEntity",
      "targetDatabase" : "data-hub-FINAL",
      "collections" : [ "MyMatchingStep", "MyPersonEntity" ],
      "additionalCollections" : [],
      "provenanceGranularityLevel" : "fine",
      "matchOptions" : { ...}
    }
  },

   "matchOptions" : {
    "dataFormat" : "json",
    "propertyDefs" : {
      "property" : [
        {
          "name" : "ssn",
          "namespace" : "",
          "localname" : "IdentificationID"
        },
      ]
    },
    "algorithms" : {
      "algorithm" : [
        {
          "name" : "std-reduce",
          "function" : "standard-reduction",
          "namespace" : "",
          "at" : ""
        },
      ]
    },
    "collections" : {
      "content" : [ "my-content-collection" ]
    },
    "scoring" : {
      "add" : [
        {
          "propertyName" : "ssn",
          "weight" : "50"
        },
      ],
      "expand" : [
        {
          "propertyName" : "first-name",
          "algorithmRef" : "thesaurus",
          "weight" : "6",
          "thesaurus" : "/mdm/config/thesauri/first-name-synonyms.xml"
        },
        {
          "propertyName" : "last-name",
          "algorithmRef" : "dbl-metaphone",
          "weight" : "8",
          "dictionary" : "name-dictionary.xml",
          "distanceThreshold" : "50"
        }
      ],
      "reduce" : [
        {
          "algorithmRef" : "std-reduce",
          "weight" : "4",
          "allMatch" : { "property" : ["last-name", "addr1"] }
        }
      ]
    },
    "actions" : {
      "action" : [
        {
          "name" : "my-custom-action",
          "function" : "custom-action",
          "namespace" : "http://marklogic.com/smart-mastering/action",
          "at" : "/custom-action.xqy"
        }
      ]
    },
    "thresholds" : {
      "threshold" : [
        { "above" : "30", "label" : "Possible Match" },
        { "above" : "50", "label" : "Likely Match", "action" : "notify" },
        { "above" : "68", "label" : "Definitive Match", "action" : "merge" },
        { "above" : "75", "label" : "Custom Match", "action" : "my-custom-action" }
      ]
    },
    "tuning" : {
      "maxScan" : 200
    }
  },

マージングステップの設定

   "4" : {
    "name" : "MyMergingStep",
    "description" : "This is my merging step.",
    "stepDefinitionName" : "default-merging",
    "stepDefinitionType" : "MERGING",
    "batchSize" : 100,
    "threadCount" : 4,
    "customHook" : {
      "module" : "/custom-modules/your-step-type/your-hook-directory/your-hook-module-name.sjs",
      "parameters" : {},
      "user" : "flow-operator",
      "runBefore" : false
    },
    "options" : {
      "stepUpdate" : false,
      "acceptsBatch" : false,
      "sourceDatabase" : "data-hub-FINAL",
      "sourceCollection" : "MyMatchingStep",
      "sourceQuery" : "cts.collectionQuery('my-custom-query')",
      "constrainSourceQueryToJob" : false,
      "targetEntity" : "MyEntity",
      "targetDatabase" : "data-hub-FINAL",
      "collections" : [ "MyMergingStep", "MyPersonEntity" ],
      "additionalCollections" : [],
      "outputFormat" : "json",
      "provenanceGranularityLevel" : "fine",
      "mergeOptions" : { ...}
    }
  },

   "mergeOptions" : {
    "matchOptions" : "mlw-match",
    "propertyDefs" : {
      "properties" : [
        {
          "name" : "ssn",
          "localname" : "IdentificationID",
          "namespace" : ""
        },
        {
          "name" : "shallow",
          "path" : "/es:envelope/es:headers/shallow"
        }
      ],
      "namespaces" : {
        "has" : "has",
        "m" : "http://marklogic.com/smart-mastering/merging",
        "es" : "http://marklogic.com/entity-services"
      }
    },
    "algorithms" : {
      "stdAlgorithm" : {
        "timestamp" : { "path" : "/es:envelope/es:headers/sm:sources/sm:source/sm:dateTime" },
        "namespaces" : {
          "sm" : "http://marklogic.com/smart-mastering",
          "es" : "http://marklogic.com/entity-services"
        }
      },
      "custom" : [
        {
          "name" : "customMerge",
          "function" : "doCustomMerge",
          "namespace" : "http://marklogic.com/smart-mastering/merging",
          "at" : "/custom-merge-xqy.xqy"
        }
      ],
      "collections" : {
        "onMerge" : {
          "function" : "collections",
          "namespace" : "test/merge-collection-algorithm",
          "at" : "/test/suites/customizing-collections/lib/merged-collections.xqy"
        },
        "onArchive" : {
          "remove" : { "collection" : ["Entity"] },
          "add" : { "collection" : ["custom-archived"] }
        },
        "onNoMatch" : {
          "function" : "noMatchCollections",
          "namespace" : "",
          "at" : "/test/suites/customizing-collections/lib/noMatchCollections.sjs"
        },
        "onNotification" : {
          "set" : { "collection" : ["notification"] }
        }
      }
    },
    "mergeStrategies" : [
      {
        "name" : "crm-source-weight",
        "algorithmRef" : "standard",
        "sourceWeights" : [
          {
            "source" : {
              "name" : "CRM",
              "weight" : "10"
            }
          }
        ]
      },
      {
        "name" : "length-weight",
        "algorithmRef" : "standard",
        "maxValues" : "1",
        "length" : { "weight" : "10" }
      }
    ],
    "merging" : [
      {
        "propertyName" : "ssn",
        "maxValues" : "1",
        "maxSources" : "1",
        "strategy" : "crm-source-weight"
      },
      {
        "propertyName" : "name",
        "maxValues" : "1",
        "doubleMetaphone" : {
          "distanceThreshold" : "50"
        },
        "synonymsSupport" : "true",
        "thesaurus" : "/mdm/config/thesauri/first-name-synonyms.xml",
        "length" : { "weight" : "8" }
      },
      {
        "propertyName" : "dob",
        "maxValues" : "1",
        "algorithmRef" : "standard",
        "sourceWeights" : {
          "source" : {
            "name" : "better-source",
            "weight" : "4"
          }
        }
      },
      {
        "default" : "true",
        "strategy" : "crm-source-weight"
      }
    ],
    "tripleMerge" : {
      "function" : "custom-trips",
      "namespace" : "http://marklogic.com/smart-mastering/merging",
      "at" : "/custom-triple-merge.xqy",
      "some-param" : 3
    }
  }

マスタリングステップの設定

   "5" : {
    "name" : "MyMasteringStep",
    "description" : "This is my mastering step.",
    "stepDefinitionName" : "default-mastering",
    "stepDefinitionType" : "MASTERING",
    "batchSize" : 100,
    "threadCount" : "1",
    "customHook" : {
      "module" : "/custom-modules/your-step-type/your-hook-directory/your-hook-module-name.sjs",
      "parameters" : {},
      "user" : "flow-operator",
      "runBefore" : false
    },
    "options" : {
      "stepUpdate" : false,
      "acceptsBatch" : false,
      "sourceDatabase" : "data-hub-FINAL",
      "sourceCollection" : "MyMappingStep",
      "sourceQuery" : "cts.collectionQuery('my-custom-query')",
      "constrainSourceQueryToJob" : false,
      "targetEntity" : "MyEntity",
      "targetDatabase" : "data-hub-FINAL",
      "collections" : [ "default-mastering, mastered" ],
      "additionalCollections" : [],
      "outputFormat" : "json",
      "provenanceGranularityLevel" : "fine",
      "matchOptions" : { ...},
      "mergeOptions" : { ...}
    }
  },

マッチングステップセクションのマッチングオプションを参照してください。

マージングステップセクションのマージングオプションを参照してください。

カスタムステップの設定

   "9" : {
    "name" : "MyCustomOtherStep",
    "description" : "This is my custom-other step.",
    "stepDefinitionName" : "custom-step-def",
    "stepDefinitionType" : "CUSTOM",
    "batchSize" : 100,
    "threadCount" : 4,
    "customHook" : {
      "module" : "/custom-modules/your-step-type/your-hook-directory/your-hook-module-name.sjs",
      "parameters" : {},
      "user" : "flow-operator",
      "runBefore" : false
    },
    "options" : {
      "stepUpdate" : false,
      "acceptsBatch" : false,
      "sourceDatabase" : "data-hub-STAGING",
      "sourceCollection" : "my-collection-tag",
      "sourceQuery" : "cts.collectionQuery('my-custom-query')",
      "constrainSourceQueryToJob" : false,
      "targetEntity" : "MyEntity",
      "targetDatabase" : "data-hub-FINAL",
      "collections" : [ "my-collection-tag" ],
      "additionalCollections" : [],
      "outputFormat" : 「json"
    }
  }