Keeping Data throughout a Pipeline

  • Published on Apr 22, 2025
  • 3 minute(s) read
  • JW
 Prev Next

The Keep Input option on every zSYNC worker is what allows for persisting and aggregating of data across several pipeline workers, or throughout an entire pipeline.

By default, Start and Stop workers are the only workers which pass their input data unchanged onto the next pipeline step. All other workers drop their input data, favouring . However, the Keep Input option in the Common Worker Config allows certain data to remain in the pipeline without being dropped.

Configuration

Click on the whitespace of the worker card to open the Common Worker Config.

Then scroll to find and modify the Keep Input config. The field can be filled with a list of comma-separated options: configuring new names for the kept data, or using the existing attribute names by default.

Example

In this scenario, several web workers are used sequentially to retrieve data from a complex HTTP API which provides Bill of Material information (see full example here). One of the pipeline workers, given a list of BOM document IDs and the associated version IDs, must fetch the related BOM element information and pass all three sets of data to the worker which follows.

Input data

//Zebroo set of BOM document version info. What's important is the document ID and version ID.

ZebrooSet:[
    {
        "metadataWorkspaceId": "00000000-0000-4000-8000-000000000000",
        "purpose": 0,
        "parent": "",
        "type": "version",
        "creator": {
            "jsonType": "user-summary",
            "state": 1,
            "isOnshapeSupport": false,
            "image": "/images/placeholder-user.png",
            "name": "J.Ortner",
            "id": "00000000-0000-4000-8000-000000000001",
            "href": "https://my.site.com/api/v10/users/00000000-0000-4000-8000-000000000001"
        },
        "lastModifier": null,
        "description": "",
        "modifiedAt": "2025-04-15T09:46:43.937+00:00",
        "thumbnail": null,
        "documentId": "00000000-0000-4000-8000-000000000d0c", //we need this
        "createdAt": "2025-04-15T09:46:39.523+00:00",
        "microversion": "00000000-0000-4000-8000-000000000ccc",
        "parents": null,
        "overrideDate": null,
        "name": "Start",
        "id": "00000000-0000-4000-8000-000000000111", //we need this
        "href": null
    }
    //...
]

Kept data

We configure the worker to keep the required input data, preventing it’s overwrite with whatever the specific worker does (in this case, fetching API data).

{'documentId': record.documentId},{'versionId': record.id}

Fetched API data

//A request is made for each incoming record and has a similar response.
//This shows a single record's data.
{
    "name": "Assembly 1",
    "id": "00000000-0000-4000-8000-000000000eee",
    "type": "Assembly",
    "elementType": "ASSEMBLY",
    "thumbnailInfo": null,
    "thumbnails": null,
    "microversionId": "00000000-0000-4000-8000-000000000ddd",
    "dataType": "onshape/assembly",
    "filename": null,
    "applicationTarget": null,
    "foreignDataId": null,
    "unupdatable": false,
    "safeToShow": true,
    "specifiedUnit": null,
    "prettyType": null,
    "zip": null,
    "lengthUnits": "millimeter",
    "angleUnits": "degree",
    "massUnits": "gram",
    "timeUnits": "second",
    "forceUnits": "poundForce",
    "pressureUnits": "poundPerSquareInch",
    "momentUnits": "inchPound",
    "accelerationUnits": "millimeterPerSecondSquared",
    "angularVelocityUnits": "degreePerSecond",
    "energyUnits": "footPoundForce",
    "areaUnits": "squareMillimeter",
    "volumeUnits": "cubicMillimeter"
    //no document ID or version ID in the response data is why we need to keep it from the previous worker
}
//...

Output data

The worker combines the fetched API data with the input data fields specified in the kept input config.

ZebrooSet:[
    {
        //Fetched API data
        "name": "Assembly 1",
        "id": "00000000-0000-4000-8000-000000000eee",
        "type": "Assembly",
        "elementType": "ASSEMBLY",
        "thumbnailInfo": null,
        "thumbnails": null,
        "microversionId": "00000000-0000-4000-8000-000000000ddd",
        "dataType": "onshape/assembly",
        "filename": null,
        "applicationTarget": null,
        "foreignDataId": null,
        "unupdatable": false,
        "safeToShow": true,
        "specifiedUnit": null,
        "prettyType": null,
        "zip": null,
        "lengthUnits": "millimeter",
        "angleUnits": "degree",
        "massUnits": "gram",
        "timeUnits": "second",
        "forceUnits": "poundForce",
        "pressureUnits": "poundPerSquareInch",
        "momentUnits": "inchPound",
        "accelerationUnits": "millimeterPerSecondSquared",
        "angularVelocityUnits": "degreePerSecond",
        "energyUnits": "footPoundForce",
        "areaUnits": "squareMillimeter",
        "volumeUnits": "cubicMillimeter",
        //Kept Input
        "documentId": "00000000-0000-4000-8000-000000000d0c",
        "versionId": "00000000-0000-4000-8000-000000000111"
    }
    //...
]

The result is that data is aggregated throughout the pipeline instead of overwritten/lost.

Related articles