public class ContentWriter<VALUEOUT> extends MarkLogicRecordWriter<DocumentURI,VALUEOUT> implements MarkLogicConstants
Modifier and Type | Field and Description |
---|---|
protected AssignmentManager |
am |
protected int |
batchSize
Batch size.
|
protected boolean[] |
blacklist
An array of blacklist host
|
protected List<DocumentURI>[] |
commitUris
URIs of the documents to be committed.
|
protected boolean |
countBased |
protected int[] |
counts
Counts of documents per forest.
|
protected int[] |
curReplica
An array of current replica
|
protected long |
effectiveVersion |
protected int |
failed |
protected boolean |
fastLoad
Whether in fast load mode.
|
protected
com.marklogic.xcc.Content[][] |
forestContents
Content lists for each forest.
|
protected String[] |
forestIds
An array of forest/host ids
|
protected int |
hostId |
protected Map<String,com.marklogic.xcc.ContentSource> |
hostSourceMap
A map from a host to a ContentSource.
|
static String |
ID_PREFIX |
protected InputStream |
is |
protected boolean |
isCopyColls |
protected boolean |
isCopyMeta |
protected boolean |
isCopyQuality |
protected boolean |
isTxnCompatible |
static
org.apache.commons.logging.Log |
LOG |
protected int |
maxRetries |
protected int |
maxSleepTime |
protected boolean |
needCommit |
protected
com.marklogic.xcc.ContentCreateOptions |
options
Content options of the output documents.
|
protected String |
outputDir
Directory of the output documents.
|
protected HashMap<com.marklogic.xcc.Content,DocumentURI>[] |
pendingUris
URIs of the documents in a batch.
|
protected HashMap<String,com.marklogic.xcc.ContentPermission[]> |
permsMap |
protected int |
retry
for failover retry
|
protected LinkedMapWritable |
roleMap
role-id -> role-name mapping
|
protected
com.marklogic.xcc.Session[] |
sessions
Sessions per forest.
|
protected int |
sfId |
protected int |
sleepTime |
protected int[] |
stmtCounts
Counts of requests per forest.
|
protected int |
succeeded |
conf,
hostName,
txnSize
ADVANCED_MODE,
ASSIGNMENT_POLICY,
BASIC_MODE,
BATCH_SIZE,
BIND_SPLIT_RANGE,
COLLECTION_FILTER,
CONTENT_TYPE,
COPY_COLLECTIONS,
COPY_METADATA,
COPY_QUALITY,
DEFAULT_BATCH_SIZE,
DEFAULT_CONTENT_TYPE,
DEFAULT_LOCAL_MAX_SPLIT_SIZE,
DEFAULT_MAX_SPLIT_SIZE,
DEFAULT_OUTPUT_CONTENT_ENCODING,
DEFAULT_OUTPUT_XML_REPAIR_LEVEL,
DEFAULT_PROPERTY_OPERATION_TYPE,
DEFAULT_TXN_SIZE,
DIRECTORY_FILTER,
DOCUMENT_SELECTOR,
EXECUTION_MODE,
EXTRACT_URI,
INDENTED,
INPUT_DATABASE_NAME,
INPUT_HOST,
INPUT_KEY_CLASS,
INPUT_LEXICON_FUNCTION_CLASS,
INPUT_MODE,
INPUT_PASSWORD,
INPUT_PORT,
INPUT_QUERY,
INPUT_QUERY_LANGUAGE,
INPUT_QUERY_TIMESTAMP,
INPUT_RESTRICT_HOSTS,
INPUT_SSL_OPTIONS_CLASS,
INPUT_SSL_PROTOCOL,
INPUT_USE_SSL,
INPUT_USERNAME,
INPUT_VALUE_CLASS,
MAX_SPLIT_SIZE,
MIN_NODEUPDATE_VERSION,
MODE_DISTRIBUTED,
MODE_LOCAL,
MR_NAMESPACE,
NODE_OPERATION_TYPE,
OUTPUT_CLEAN_DIR,
OUTPUT_COLLECTION,
OUTPUT_CONTENT_ENCODING,
OUTPUT_CONTENT_LANGUAGE,
OUTPUT_CONTENT_NAMESPACE,
OUTPUT_DATABASE_NAME,
OUTPUT_DIRECTORY,
OUTPUT_FAST_LOAD,
OUTPUT_FOREST_HOST,
OUTPUT_GRAPH,
OUTPUT_HOST,
OUTPUT_KEY_TYPE,
OUTPUT_KEY_VARNAME,
OUTPUT_NAMESPACE,
OUTPUT_OVERRIDE_GRAPH,
OUTPUT_PARTITION,
OUTPUT_PASSWORD,
OUTPUT_PERMISSION,
OUTPUT_PORT,
OUTPUT_PROPERTY_ALWAYS_CREATE,
OUTPUT_QUALITY,
OUTPUT_QUERY,
OUTPUT_QUERY_LANGUAGE,
OUTPUT_RESTRICT_HOSTS,
OUTPUT_SSL_OPTIONS_CLASS,
OUTPUT_SSL_PROTOCOL,
OUTPUT_STREAMING,
OUTPUT_URI_PREFIX,
OUTPUT_URI_REPLACE,
OUTPUT_URI_SUFFIX,
OUTPUT_USE_SSL,
OUTPUT_USERNAME,
OUTPUT_VALUE_TYPE,
OUTPUT_VALUE_VARNAME,
OUTPUT_XML_REPAIR_LEVEL,
PATH_NAMESPACE,
PROPERTY_OPERATION_TYPE,
QUERY_FILTER,
RECORD_TO_FRAGMENT_RATIO,
REDACTION_RULE_COLLECTION,
SPLIT_END_VARNAME,
SPLIT_QUERY,
SPLIT_START_VARNAME,
SUBDOCUMENT_EXPRESSION,
TEMPORAL_COLLECTION,
TXN_SIZE,
TYPE_FILTER
Constructor and Description |
---|
ContentWriter(org.apache.hadoop.conf.Configuration conf,
Map<String,com.marklogic.xcc.ContentSource> hostSourceMap,
boolean fastLoad) |
ContentWriter(org.apache.hadoop.conf.Configuration conf,
Map<String,com.marklogic.xcc.ContentSource> hostSourceMap,
boolean fastLoad, AssignmentManager am) |
Modifier and Type | Method and Description |
---|---|
void |
close(org.apache.hadoop.mapreduce.TaskAttemptContext context) |
protected void |
commit(int id) |
protected
com.marklogic.xcc.Content |
createContent(DocumentURI key,
VALUEOUT value) |
protected
com.marklogic.xcc.Session |
getSession(int fId,
boolean nextReplica) |
protected
com.marklogic.xcc.Session |
getSession(int fId,
boolean nextReplica,
com.marklogic.xcc.Session.TransactionMode mode) |
int |
getTransactionSize(org.apache.hadoop.conf.Configuration conf) |
protected void |
insertBatch(com.marklogic.xcc.Content[] batch,
int id)
Insert batch, log errors and update stats.
|
protected boolean |
needCommit() |
protected void |
rollback(int id) |
protected void |
rollbackCount(int fId) |
void |
write(DocumentURI key,
VALUEOUT value) |
commitIfNecessary,
getSession
public static final org.apache.commons.logging.Log LOG
public static final String ID_PREFIX
protected String outputDir
protected com.marklogic.xcc.ContentCreateOptions options
protected Map<String,com.marklogic.xcc.ContentSource> hostSourceMap
protected com.marklogic.xcc.Content[][] forestContents
protected String[] forestIds
protected int[] curReplica
protected boolean[] blacklist
protected int[] counts
protected HashMap<com.marklogic.xcc.Content,DocumentURI>[] pendingUris
protected List<DocumentURI>[] commitUris
protected boolean fastLoad
protected int batchSize
protected int[] stmtCounts
protected com.marklogic.xcc.Session[] sessions
protected InputStream is
protected AssignmentManager am
protected int sfId
protected boolean countBased
protected LinkedMapWritable roleMap
protected int succeeded
protected int failed
protected boolean needCommit
protected int hostId
protected boolean isCopyColls
protected boolean isCopyQuality
protected boolean isCopyMeta
protected long effectiveVersion
protected boolean isTxnCompatible
protected int retry
protected final int maxRetries
protected int sleepTime
protected final int maxSleepTime
public ContentWriter(org.apache.hadoop.conf.Configuration conf, Map<String,com.marklogic.xcc.ContentSource> hostSourceMap, boolean fastLoad)
public ContentWriter(org.apache.hadoop.conf.Configuration conf, Map<String,com.marklogic.xcc.ContentSource> hostSourceMap, boolean fastLoad, AssignmentManager am)
protected boolean needCommit()
protected com.marklogic.xcc.Content createContent(DocumentURI key, VALUEOUT value) throws IOException
IOException
protected void insertBatch(com.marklogic.xcc.Content[] batch, int id) throws IOException
batch
- batch of content to insertid
- forest IdIOException
protected void rollback(int id) throws IOException
IOException
protected void commit(int id) throws IOException
IOException
public void write(DocumentURI key, VALUEOUT value) throws IOException, InterruptedException
write
in
class org.apache.hadoop.mapreduce.RecordWriter<DocumentURI,VALUEOUT>
IOException
InterruptedException
protected void rollbackCount(int fId)
protected com.marklogic.xcc.Session getSession(int fId, boolean nextReplica, com.marklogic.xcc.Session.TransactionMode mode)
protected com.marklogic.xcc.Session getSession(int fId, boolean nextReplica)
public void close(org.apache.hadoop.mapreduce.TaskAttemptContext context) throws IOException, InterruptedException
close
in class MarkLogicRecordWriter<DocumentURI,VALUEOUT>
IOException
InterruptedException
public int getTransactionSize(org.apache.hadoop.conf.Configuration conf)
getTransactionSize
in class MarkLogicRecordWriter<DocumentURI,VALUEOUT>
Copyright © 2020 MarkLogic
Corporation. All Rights Reserved.
Complete online documentation for MarkLogic Server,
XQuery and related components may be found at
developer.marklogic.com