Create a Flow Using Gradle
Before you begin
You need:
- Java SE JDK 8
- MarkLogic Server (See Version Compatibility.)
- Gradle 4.6 or later
About this task
After creating a project, the next step is to create a flow.
Procedure
Results
View an example.
{
"name": "your-flow-name",
"description": "This is the default flow containing all of the default steps",
"batchSize": 100,
"threadCount": 4,
"options": {
"sourceQuery": null
},
"steps": {
"1": {
"name": "ingestion-step",
"description": "This is the default ingestion step",
"stepDefinitionName": "default-ingestion",
"stepDefinitionType": "INGESTION",
"customHook": {},
"fileLocations": {
"inputFilePath": "path/to/folder",
"outputURIReplacement": "output/URI,'substitute/URI'",
"inputFileType": "json"
},
"options": {
"targetDatabase": "data-hub-STAGING",
"sourceQuery": null,
"outputFormat": "json",
"collections": [
"default-ingestion"
]
}
},
"2": {
"name": "mapping-step",
"description": "This is the default mapping step",
"stepDefinitionName": "default-mapping",
"stepDefinitionType": "MAPPING",
"options": {
"sourceDatabase": "data-hub-STAGING",
"targetDatabase": "data-hub-FINAL",
"sourceQuery": "cts.collectionQuery('default-ingestion')",
"collections": [
"default-mapping",
"mdm-content"
],
"targetEntity": "entity-name",
"mapping": {
"name": "mapping-name",
"version": 1
}
}
},
"3": {
"name": "mastering-step",
"description": "This is the default mastering step",
"stepDefinitionName": "default-mastering",
"stepDefinitionType": "MASTERING",
"options": {
"sourceDatabase": "data-hub-FINAL",
"targetDatabase": "data-hub-FINAL",
"targetEntity": "entity-name",
"sourceQuery": "cts.andQuery([cts.collectionQuery('default-mapping'),cts.collectionQuery('mdm-content')])",
"collections": [
"default-mastering, mastered"
],
"mergeOptions" : {
"matchOptions" : "",
"propertyDefs" : {
"properties" : [ ],
"namespaces" : { }
},
"algorithms" : {
"stdAlgorithm" : {
"timestamp" : { }
},
"custom" : [ ],
"collections" : { }
},
"mergeStrategies" : [ ],
"merging" : [ ]
},
"matchOptions" : {
"dataFormat": "json",
"propertyDefs": {
"property": []
},
"algorithms": {
"algorithm": []
},
"collections": {
"content": []
},
"scoring": {
"add": [],
"expand": [],
"reduce": []
},
"actions": {
"action": []
},
"thresholds": {
"threshold": []
},
"tuning": {
"maxScan": 200
}
}
}
}
}
}
Important: This flow will not run correctly as is. You must customize the steps based on your needs.