Skip to content

Commit ce39bf3

Browse files
lennessyylennessyy
committed
docs(external-storage): wire TypeScript snippets to snipsync
Bump snipsync to 1.13.0, which indents spliced blocks to match their SNIPSTART marker instead of writing at column 0. A fenced block at column 0 inside a numbered list closes the list item, which split the Go and Python procedures into two lists and broke the MDX build outright on the TypeScript page, where the marker sits inside a Tabs element. That is the whole change to the Go and Python pages: same content, now indented under their markers. Two Python markers were at six spaces while their SNIPEND and list content were at three, so normalize them. The TypeScript snippets were materialized by pinning the snipsync origins at temporalio/features#855 and temporalio/samples-typescript#503, then restoring the config. A run against main is a no-op, so the daily snipsync job will re-sync these byte for byte once both land. Until they do, the source files are not on main, so the generated code block titles and the links to the samples repo would 404. Strip them, along with the custom driver excerpt that came from the unmerged sample. The daily job restores the titles on its own after the sources land.
1 parent 3e48a86 commit ce39bf3

5 files changed

Lines changed: 82 additions & 110 deletions

File tree

‎docs/develop/go/best-practices/data-handling/external-storage.mdx‎

Lines changed: 32 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -42,43 +42,43 @@ The Go SDK includes an S3 storage driver. Follow these steps to set it up:
4242
1. Load your AWS configuration and create the S3 storage driver. The driver uses your standard [AWS credentials](https://docs.aws.amazon.com/sdk-for-go/v2/developer-guide/configure-gosdk.html) from the environment (environment variables, IAM role, or AWS config file):
4343

4444
<!--SNIPSTART go-s3-driver-create-->
45-
[features/snippets/external_storage/s3_setup/s3_driver_create.go](https://github.com/temporalio/features/blob/main/features/snippets/external_storage/s3_setup/s3_driver_create.go)
46-
```go
47-
cfg, err := config.LoadDefaultConfig(context.Background(),
48-
config.WithRegion("us-east-2"),
49-
)
50-
if err != nil {
51-
log.Fatalf("load AWS config: %v", err)
52-
}
53-
54-
driver, err := s3driver.NewDriver(s3driver.Options{
55-
Client: awssdkv2.NewClient(s3.NewFromConfig(cfg)),
56-
Bucket: s3driver.StaticBucket("my-temporal-payloads"),
57-
})
58-
if err != nil {
59-
log.Fatalf("create S3 driver: %v", err)
60-
}
61-
```
45+
[features/snippets/external_storage/s3_setup/s3_driver_create.go](https://github.com/temporalio/features/blob/main/features/snippets/external_storage/s3_setup/s3_driver_create.go)
46+
```go
47+
cfg, err := config.LoadDefaultConfig(context.Background(),
48+
config.WithRegion("us-east-2"),
49+
)
50+
if err != nil {
51+
log.Fatalf("load AWS config: %v", err)
52+
}
53+
54+
driver, err := s3driver.NewDriver(s3driver.Options{
55+
Client: awssdkv2.NewClient(s3.NewFromConfig(cfg)),
56+
Bucket: s3driver.StaticBucket("my-temporal-payloads"),
57+
})
58+
if err != nil {
59+
log.Fatalf("create S3 driver: %v", err)
60+
}
61+
```
6262
<!--SNIPEND-->
6363

6464
2. Configure the driver on `ExternalStorage` and pass it in your Client options:
6565

6666
<!--SNIPSTART go-s3-external-storage-setup-->
67-
[features/snippets/external_storage/s3_setup/s3_external_storage_setup.go](https://github.com/temporalio/features/blob/main/features/snippets/external_storage/s3_setup/s3_external_storage_setup.go)
68-
```go
69-
c, err := client.Dial(client.Options{
70-
HostPort: "localhost:7233",
71-
ExternalStorage: converter.ExternalStorage{
72-
Drivers: []converter.StorageDriver{driver},
73-
},
74-
})
75-
if err != nil {
76-
log.Fatalf("connect to Temporal: %v", err)
77-
}
78-
defer c.Close()
79-
80-
w := worker.New(c, "my-task-queue", worker.Options{})
81-
```
67+
[features/snippets/external_storage/s3_setup/s3_external_storage_setup.go](https://github.com/temporalio/features/blob/main/features/snippets/external_storage/s3_setup/s3_external_storage_setup.go)
68+
```go
69+
c, err := client.Dial(client.Options{
70+
HostPort: "localhost:7233",
71+
ExternalStorage: converter.ExternalStorage{
72+
Drivers: []converter.StorageDriver{driver},
73+
},
74+
})
75+
if err != nil {
76+
log.Fatalf("connect to Temporal: %v", err)
77+
}
78+
defer c.Close()
79+
80+
w := worker.New(c, "my-task-queue", worker.Options{})
81+
```
8282
<!--SNIPEND-->
8383

8484
By default, payloads of 256 KiB or larger are offloaded to external storage. You can adjust this with the

‎docs/develop/python/best-practices/data-handling/external-storage.mdx‎

Lines changed: 29 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -43,39 +43,39 @@ The Python SDK includes an S3 storage driver. Follow these steps to set it up:
4343
1. Create an S3 client using `aioboto3` and pass it to the `S3StorageDriver`. The driver uses your standard AWS
4444
credentials from the environment (environment variables, IAM role, or AWS config file):
4545

46-
<!--SNIPSTART python-s3-driver-create-->
47-
[features/snippets/external_storage/s3_setup/s3_driver_create.py](https://github.com/temporalio/features/blob/main/features/snippets/external_storage/s3_setup/s3_driver_create.py)
48-
```py
49-
session = aioboto3.Session(profile_name=AWS_PROFILE, region_name=AWS_REGION)
50-
async with session.client("s3") as s3_client:
51-
driver = S3StorageDriver(
52-
client=new_aioboto3_client(s3_client),
53-
bucket="my-temporal-payloads",
54-
)
55-
```
46+
<!--SNIPSTART python-s3-driver-create-->
47+
[features/snippets/external_storage/s3_setup/s3_driver_create.py](https://github.com/temporalio/features/blob/main/features/snippets/external_storage/s3_setup/s3_driver_create.py)
48+
```py
49+
session = aioboto3.Session(profile_name=AWS_PROFILE, region_name=AWS_REGION)
50+
async with session.client("s3") as s3_client:
51+
driver = S3StorageDriver(
52+
client=new_aioboto3_client(s3_client),
53+
bucket="my-temporal-payloads",
54+
)
55+
```
5656
<!--SNIPEND-->
5757

5858
2. Configure the driver on your `DataConverter` and pass the converter to your Client and Worker:
5959

60-
<!--SNIPSTART python-s3-external-storage-setup-->
61-
[features/snippets/external_storage/s3_setup/s3_external_storage_setup.py](https://github.com/temporalio/features/blob/main/features/snippets/external_storage/s3_setup/s3_external_storage_setup.py)
62-
```py
63-
data_converter = dataclasses.replace(
64-
DataConverter.default,
65-
external_storage=ExternalStorage(drivers=[driver]),
66-
)
67-
68-
client_config = ClientConfig.load_client_connect_config()
69-
70-
client = await Client.connect(**client_config, data_converter=data_converter)
71-
72-
worker = Worker(
73-
client,
74-
task_queue="my-task-queue",
75-
workflows=[],
76-
activities=[],
77-
)
78-
```
60+
<!--SNIPSTART python-s3-external-storage-setup-->
61+
[features/snippets/external_storage/s3_setup/s3_external_storage_setup.py](https://github.com/temporalio/features/blob/main/features/snippets/external_storage/s3_setup/s3_external_storage_setup.py)
62+
```py
63+
data_converter = dataclasses.replace(
64+
DataConverter.default,
65+
external_storage=ExternalStorage(drivers=[driver]),
66+
)
67+
68+
client_config = ClientConfig.load_client_connect_config()
69+
70+
client = await Client.connect(**client_config, data_converter=data_converter)
71+
72+
worker = Worker(
73+
client,
74+
task_queue="my-task-queue",
75+
workflows=[],
76+
activities=[],
77+
)
78+
```
7979
<!--SNIPEND-->
8080

8181
By default, payloads of 256 KiB or larger are offloaded to external storage. You can adjust this with the

‎docs/develop/typescript/best-practices/data-handling/external-storage.mdx‎

Lines changed: 16 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -123,18 +123,20 @@ tabs that follow. Only the driver setup differs between the two. Everything afte
123123
import { ExternalStorage } from '@temporalio/common';
124124
import { Worker } from '@temporalio/worker';
125125

126-
const dataConverter = {
127-
externalStorage: new ExternalStorage({ drivers: [driver] }),
128-
};
129-
130-
const connection = await Connection.connect();
131-
const client = new Client({ connection, dataConverter });
132-
133-
const worker = await Worker.create({
134-
workflowsPath: require.resolve('./workflows'),
135-
taskQueue: 'my-task-queue',
136-
dataConverter,
137-
});
126+
async function createClientAndWorker() {
127+
const dataConverter = {
128+
externalStorage: new ExternalStorage({ drivers: [driver] }),
129+
};
130+
131+
const connection = await Connection.connect();
132+
const client = new Client({ connection, dataConverter });
133+
134+
const worker = await Worker.create({
135+
workflowsPath: require.resolve('./workflows'),
136+
taskQueue: 'my-task-queue',
137+
dataConverter,
138+
});
139+
}
138140
```
139141
<!--SNIPEND-->
140142

@@ -154,39 +156,10 @@ If you need a storage backend other than what the built-in drivers allow, you ca
154156
Refer to [Choose a storage system](/external-storage#choose-storage) for guidance on selecting a backing store and
155157
[Lifecycle management](/external-storage#lifecycle) for retention requirements.
156158

157-
The [external-storage sample](https://github.com/temporalio/samples-typescript/tree/main/external-storage) implements a
158-
complete driver that keeps payloads on the filesystem, with tests that exercise it end to end. Its `store` and
159-
`retrieve` methods show the shape every driver has. Each one fans out over the batch it was handed, returns results in
160-
the order it received them, and passes the SDK's abort signal down so that one failure cancels its siblings:
161-
162-
<!--SNIPSTART typescript-custom-storage-driver -->
163-
[external-storage/src/filesystem-storage-driver.ts](https://github.com/temporalio/samples-typescript/blob/main/external-storage/src/filesystem-storage-driver.ts)
164-
```ts
165-
async store(context: StorageDriverStoreContext, payloads: Payload[]): Promise<StorageDriverClaim[]> {
166-
const keyPrefix = buildKeyPrefix(context.target);
167-
return runAllAbortingOnFirstError(context.abortSignal, (signal) =>
168-
payloads.map((payload) => this.storePayload(payload, keyPrefix, signal)),
169-
);
170-
}
171-
172-
/** Inverse of {@link store}: one payload per claim, in the same order. */
173-
async retrieve(context: StorageDriverRetrieveContext, claims: StorageDriverClaim[]): Promise<Payload[]> {
174-
return runAllAbortingOnFirstError(context.abortSignal, (signal) =>
175-
claims.map((claim) => this.retrievePayload(claim, signal)),
176-
);
177-
}
178-
```
179-
<!--SNIPEND-->
180-
181-
The per-payload work happens in the private `storePayload` and `retrievePayload` methods. Read
182-
[the full driver](https://github.com/temporalio/samples-typescript/blob/main/external-storage/src/filesystem-storage-driver.ts)
183-
for the parts this page only describes: content-addressed keys, an atomic write, a hash check on read, and a guard that
184-
rejects claims resolving outside the storage root.
185-
186159
A shared filesystem works for local development and for Workers that mount the same volume. For anything else, use a
187160
storage system that every Client and Worker can reach.
188161

189-
The following sections walk through the key parts of the driver implementation.
162+
The following sections walk through the key parts of a custom driver.
190163

191164
### 1. Implement the StorageDriver interface
192165

@@ -198,7 +171,7 @@ A custom driver implements the `StorageDriver` interface, which has two readonly
198171
- `type` is a string that identifies the driver implementation, and the Worker reports it in its heartbeat. Unlike
199172
`name`, `type` must be the same across all instances of the same driver type regardless of configuration. Two S3
200173
drivers named `"s3-primary"` and `"s3-archive"` would both report `"aws.s3driver"` as their type, while the built-in
201-
GCS driver reports `"gcp.gcsdriver"`. The filesystem driver in the sample reports `"sample.filesystemdriver"`.
174+
GCS driver reports `"gcp.gcsdriver"`.
202175
- `store()` receives an array of payloads and returns one `StorageDriverClaim` per payload. A claim wraps a set of
203176
string key-value pairs that the driver uses to locate the payload later.
204177
- `retrieve()` receives the claims that `store()` produced and returns the original payloads.
@@ -230,7 +203,6 @@ Client and Worker. Both sides need it: a Client without External Storage configu
230203
can also package your driver as a [plugin](/develop/plugins-guide) for easier reuse across services:
231204

232205
<!--SNIPSTART typescript-custom-driver-data-converter -->
233-
[external-storage/src/data-converter.ts](https://github.com/temporalio/samples-typescript/blob/main/external-storage/src/data-converter.ts)
234206
```ts
235207
export function createDataConverter(rootDir: string = STORAGE_ROOT): DataConverter {
236208
return {

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@
8888
"satori": "^0.26.0",
8989
"search-insights": "^2",
9090
"sharp": "^0.35.3",
91-
"snipsync": "^1.12.0",
91+
"snipsync": "^1.13.0",
9292
"uuid": "^14.0.1",
9393
"wait-on": "^9.0.10",
9494
"yaml": "^2.9.0"

‎yarn.lock‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13023,10 +13023,10 @@ snake-case@^3.0.4:
1302313023
dot-case "^3.0.4"
1302413024
tslib "^2.0.3"
1302513025

13026-
snipsync@^1.12.0:
13027-
version "1.12.0"
13028-
resolved "https://registry.npmjs.org/snipsync/-/snipsync-1.12.0.tgz"
13029-
integrity sha512-UHvct6DAVjCu2y+5/tSNKIbdGoANlpUICQD+QDXyGUuX9nH6QcepRinNoCDDnaBYTvLcs7qDozp+nw0WY4E2DA==
13026+
snipsync@^1.13.0:
13027+
version "1.13.0"
13028+
resolved "https://registry.yarnpkg.com/snipsync/-/snipsync-1.13.0.tgz#4d009edc952427b68b2b514c81abb48f1e1a1c73"
13029+
integrity sha512-9S0w6CYZX38KQfTby5KWeOZs/hzB3gxxD2y8Gid2TydzmHamXH+HkBhJvtDYTt7sRsHHUCGZtA0SmNV0LGqVVw==
1303013030
dependencies:
1303113031
"@octokit/rest" "^18.12.0"
1303213032
anzip "^0.2.0"

0 commit comments

Comments
 (0)