Skip to content

Commit d8e7884

Browse files
ccollinsuahccollinsuah
committed
Added schemas and populated readme
1 parent 1c89a94 commit d8e7884

3 files changed

Lines changed: 266 additions & 4 deletions

File tree

‎tasks/add-input-granules/README.md‎

Lines changed: 67 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,67 @@
1-
# Add input granules task
2-
## Purpose
3-
This task will add the `input_granules` field to the cumulus output message. It uses the execution
4-
lists (running, failed, completed) in the payload to gather the granules
1+
# @cumulus/add-input-granules
2+
3+
This task adds a list of granules processed by child ingest executions to its output message. It uses the [Cumulus Python API](https://github.com/nasa/Cumulus-API-Python) to fetch granule information.
4+
5+
## Usage
6+
This task is intended to be used as part of PDR workflows. It is run after all ingest workflows are completed and prior to granule cleanup tasks to which it provides input granules.
7+
8+
### Input
9+
10+
| field name | type | default | required | values | description
11+
| ---------- | ---- | ------- | -------- | ------ | -----------
12+
| pdr | object | N/A | yes | N/A | Product Delivery Record
13+
| pdr.name | string | N/A | yes | N/A | PDR filename
14+
| pdr.path | string | N/A | yes | N/A | PDR location
15+
| pdr.archivePath | string | N/A | yes | N/A | Archived PDR location
16+
| running | array[string] | N/A | yes | N/A | List of queued and running workflow execution ARNS
17+
| completed | array[string] | N/A | yes | N/A | List of completed workflow execution ARNs
18+
| failed | array[object] | N/A | yes | N/A | List of failed workflow ARNs and reason for failure
19+
| failed[].arn | string | N/A | yes | N/A | Failed execution ARN
20+
| failed[].reason | string | N/A | yes | N/A | Reason for workflow failure
21+
22+
### Output
23+
24+
| field name | type | default | required | values | description
25+
| ---------- | ---- | ------- | -------- | ------ | -----------
26+
| pdr | object | N/A | yes | N/A | Product Delivery Record
27+
| pdr.name | string | N/A | yes | N/A | PDR filename
28+
| pdr.path | string | N/A | yes | N/A | PDR location
29+
| pdr.archivePath | string | N/A | yes | N/A | Archived PDR location
30+
| running | array[string] | N/A | yes | N/A | List of queued and running workflow execution ARNS
31+
| completed | array[string] | N/A | yes | N/A | List of completed workflow execution ARNs
32+
| failed | array[object] | N/A | yes | N/A | List of failed workflow ARNs and reason for failure
33+
| failed[].arn | string | N/A | yes | N/A | Failed execution ARN
34+
| failed[].reason | string | N/A | yes | N/A | Reason for workflow failure
35+
| granules | array[object] | N/A | yes | N/A | List of granules
36+
| granules[].granuleId | string | N/A | yes | N/A | Granule ID
37+
| granules[].files | array[object] | N/A | yes | N/A | List of files associated with granule
38+
| granules[].files[].bucket | string | N/A | yes | N/A | Bucket where file is archived in S3
39+
| granules[].files[].checksum | string | N/A | no | N/A | Checksum value for file
40+
| granules[].files[].fileName | string | N/A | no | N/A | Name of file (e.g. file.txt)
41+
| granules[].files[].key | string | N/A | yes | N/A | S3 Key for archived file
42+
| granules[].files[].size | integer | N/A | no | N/A | Size of file (in bytes)
43+
| granules[].files[].source | string | N/A | no | N/A | Source URI of the file from origin system (e.g. S3, FTP, HTTP)
44+
| granules[].files[].type | string | N/A | no | N/A | Type of file (e.g. data, metadata, browse)
45+
46+
### Example workflow configuration and use
47+
48+
49+
### Internal Dependencies
50+
51+
This task uses the Cumulus Private API Lambda via the Cumulus Python API and requires the `PRIVATE_API_LAMBDA_ARN` environment variable to be set.
52+
53+
### External Dependencies
54+
55+
- https://github.com/nasa/Cumulus-API-Python
56+
- https://github.com/nasa/cumulus-message-adapter
57+
- https://github.com/nasa/cumulus-message-adapter-python
58+
59+
## Contributing
60+
61+
To make a contribution, please [see our Cumulus contributing guidelines](https://github.com/nasa/cumulus/blob/master/CONTRIBUTING.md) and our documentation on [adding a task](https://nasa.github.io/cumulus/docs/adding-a-task)
62+
63+
## About Cumulus
64+
65+
Cumulus is a cloud-based data ingest, archive, distribution and management prototype for NASA's future Earth science data streams.
66+
67+
[Cumulus Documentation](https://nasa.github.io/cumulus)

‎tasks/add-input-granules/schemas/input.json‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
{
2+
"title": "AddInputGranulesInput",
3+
"description": "Describes the input expected by the add-input-granules task",
4+
"type": "object",
5+
"required": [
6+
"pdr",
7+
"running",
8+
"completed",
9+
"failed"
10+
],
11+
"properties": {
12+
"pdr": {
13+
"description": "Product Delivery Record",
14+
"type": "object",
15+
"required": [
16+
"name",
17+
"path",
18+
"archivePath"
19+
],
20+
"properties": {
21+
"name": {
22+
"description": "Filename of the PDR",
23+
"type": "string"
24+
},
25+
"path": {
26+
"description": "Location of the PDR",
27+
"type": "string"
28+
},
29+
"archivePath": {
30+
"description": "Archive path of the PDR",
31+
"type": "string"
32+
}
33+
}
34+
},
35+
"running": {
36+
"description": "List of execution ARNs which are queued or running",
37+
"type": "array",
38+
"items": {
39+
"type": "string"
40+
}
41+
},
42+
"completed": {
43+
"description": "List of completed execution ARNs",
44+
"type": "array",
45+
"items": {
46+
"type": "string"
47+
}
48+
},
49+
"failed": {
50+
"description": "List of failed execution ARNs with reason for failure",
51+
"type": "array",
52+
"items": {
53+
"type": "object",
54+
"required": [
55+
"arn",
56+
"reason"
57+
],
58+
"properties": {
59+
"arn": {
60+
"description": "ARN of the failed execution",
61+
"type": "string"
62+
},
63+
"reason": {
64+
"description": "Reason for the failed execution",
65+
"type": "string"
66+
}
67+
}
68+
}
69+
}
70+
}
71+
}

‎tasks/add-input-granules/schemas/output.json‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
{
2+
"title": "AddInputGranulesOutput",
3+
"description": "Describes the output produced by the add-input-granules task",
4+
"type": "object",
5+
"required": [
6+
"pdr",
7+
"running",
8+
"completed",
9+
"failed",
10+
"granules"
11+
],
12+
"properties": {
13+
"pdr": {
14+
"description": "Product Delivery Record",
15+
"type": "object",
16+
"required": [
17+
"name",
18+
"path"
19+
],
20+
"properties": {
21+
"name": {
22+
"description": "Filename of the PDR",
23+
"type": "string"
24+
},
25+
"path": {
26+
"description": "Location of the PDR",
27+
"type": "string"
28+
}
29+
}
30+
},
31+
"running": {
32+
"description": "List of execution ARNs which are queued or running",
33+
"type": "array",
34+
"items": {
35+
"type": "string"
36+
}
37+
},
38+
"completed": {
39+
"description": "List of completed execution ARNs",
40+
"type": "array",
41+
"items": {
42+
"type": "string"
43+
}
44+
},
45+
"failed": {
46+
"description": "List of failed execution ARNs with reason for failure",
47+
"type": "array",
48+
"items": {
49+
"type": "object",
50+
"required": [
51+
"arn",
52+
"reason"
53+
],
54+
"properties": {
55+
"arn": {
56+
"description": "ARN of the failed execution",
57+
"type": "string"
58+
},
59+
"reason": {
60+
"description": "Reason for the failed execution",
61+
"type": "string"
62+
}
63+
}
64+
}
65+
},
66+
"granules": {
67+
"type": "array",
68+
"description": "List of granules added by the add-input-granules task.",
69+
"items": {
70+
"type": "object",
71+
"required": [
72+
"granuleId",
73+
"files"
74+
],
75+
"properties": {
76+
"granuleId": {
77+
"type": "string"
78+
},
79+
"files": {
80+
"type": "array",
81+
"items": {
82+
"additionalProperties": false,
83+
"type": "object",
84+
"required": [
85+
"bucket",
86+
"key"
87+
],
88+
"properties": {
89+
"bucket": {
90+
"description": "Bucket where file is archived in S3",
91+
"type": "string"
92+
},
93+
"checksum": {
94+
"description": "Checksum value for file",
95+
"type": "string"
96+
},
97+
"checksumType": {
98+
"description": "Type of checksum (e.g. md5, sha256, etc)",
99+
"type": "string"
100+
},
101+
"fileName": {
102+
"description": "Name of file (e.g. file.txt)",
103+
"type": "string"
104+
},
105+
"key": {
106+
"description": "S3 Key for archived file",
107+
"type": "string"
108+
},
109+
"size": {
110+
"description": "Size of file (in bytes)",
111+
"type": "number"
112+
},
113+
"source": {
114+
"description": "Source URI of the file from origin system (e.g. S3, FTP, HTTP)",
115+
"type": "string"
116+
},
117+
"type": {
118+
"description": "Type of file (e.g. data, metadata, browse)",
119+
"type": "string"
120+
}
121+
}
122+
}
123+
}
124+
}
125+
}
126+
}
127+
}
128+
}

0 commit comments

Comments
 (0)