Skip to content

Commit c93d186

Browse files
obozhko-folioobozhko-folio
authored
MODEXPW-598 - Remove Bulk edit job references (#677)
* MODEXPW-598 Initial commit
1 parent d96fb44 commit c93d186

156 files changed

Lines changed: 46 additions & 10710 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎README.md‎

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -20,16 +20,11 @@ Other [modules](https://dev.folio.org/source-code/#server-side) are described,
2020
with further FOLIO Developer documentation at
2121
[dev.folio.org](https://dev.folio.org/)
2222

23-
### Bulk edit
24-
In case of no matched records found when uploading CSV file with items or users, link to download matched records is not available for user.
25-
The maximum value of size for uploading file is 15MB. It could be changed with spring.servlet.multipart.max-file-size application argument.
26-
2723
### Memory configuration
2824
To stable module operating the following mod-data-export-worker configuration is required: Java args -XX:MetaspaceSize=384m -XX:MaxMetaspaceSize=512m -Xmx2048m,
2925
AWS container: memory - 3072, memory (soft limit) - 2600, cpu - 1024.
3026

3127
### Environment variables
32-
This module uses separate storage of temporary (local) files for its work. These files are necessary for processing bulk-edit business flows.
3328
Any S3-compatible storage (AWS S3, Minio Server) supported by the Minio Client can be used as such storage. Thus, in addition to the
3429
AWS configuration (AWS_URL, AWS_REGION, AWS_BUCKET, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) of the permanent storage,
3530
one need to configure the environment settings for s3 subpathes (S3_SUB_PATH, S3_LOCAL_SUB_PATH).
@@ -63,9 +58,4 @@ This value should be `true` if AWS S3 is used as storage.
6358
| CHUNKS | 100 | Number of items being passed to write at once |
6459
| CORE_POOL_SIZE | 10 | Maximum number of threads being created for each task before the queue is utilized |
6560
| MAX_POOL_SIZE | 10 | Maximum number of threads that can be created after the queue is full and before rejecting the new tasks |
66-
| BUCKET_SIZE | 50 | Size of the bucket used in partitioning parameters |
67-
| QUANTITY_PER_PARTITION | 5000 | Number of records per one partition in bulk edit job |
68-
| MERGE_CSV_JSON_MRC_POOL_SIZE | 3 | Number of threads to merge csv, json and mrc part files in bulk edit job |
69-
| NUM_PARTITIONS | 8 | Number of partitions in bulk edit job |
70-
| MINUTES_FOR_MERGE | 10 | Number of minutes to wait until all csv, json and mrc part files are merged in bulk edit job |
71-
| DB_POOL_SIZE | 20 | Maximum number of database connections at once (once reached, any additional requests will be queued) |
61+
| BUCKET_SIZE | 50 | Size of the bucket used in partitioning parameters |

‎descriptors/ModuleDescriptor-template.json‎

Lines changed: 0 additions & 158 deletions
Original file line numberDiff line numberDiff line change
@@ -222,139 +222,6 @@
222222
"id": "bulk-edit",
223223
"version": "4.3",
224224
"handlers": [
225-
{
226-
"methods": [ "POST" ],
227-
"pathPattern": "/bulk-edit/{id}/upload",
228-
"permissionsRequired": [ "bulk-edit.item.post" ],
229-
"modulePermissions": [
230-
"addresstypes.collection.get",
231-
"addresstypes.item.get",
232-
"data-export.job.item.get",
233-
"departments.collection.get",
234-
"departments.item.get",
235-
"inventory-storage.call-number-types.collection.get",
236-
"inventory-storage.call-number-types.item.get",
237-
"inventory-storage.electronic-access-relationships.collection.get",
238-
"inventory-storage.electronic-access-relationships.item.get",
239-
"inventory-storage.holdings-note-types.collection.get",
240-
"inventory-storage.holdings-note-types.item.get",
241-
"inventory-storage.holdings-sources.collection.get",
242-
"inventory-storage.holdings-sources.item.get",
243-
"inventory-storage.holdings-types.collection.get",
244-
"inventory-storage.holdings-types.item.get",
245-
"inventory-storage.holdings.collection.get",
246-
"inventory-storage.holdings.item.get",
247-
"inventory-storage.ill-policies.collection.get",
248-
"inventory-storage.ill-policies.item.get",
249-
"inventory-storage.instances.collection.get",
250-
"inventory-storage.instances.item.get",
251-
"inventory-storage.item-damaged-statuses.collection.get",
252-
"inventory-storage.item-damaged-statuses.item.get",
253-
"inventory-storage.item-note-types.collection.get",
254-
"inventory-storage.item-note-types.item.get",
255-
"inventory-storage.loan-types.collection.get",
256-
"inventory-storage.loan-types.item.get",
257-
"inventory-storage.locations.collection.get",
258-
"inventory-storage.locations.item.get",
259-
"inventory-storage.material-types.collection.get",
260-
"inventory-storage.material-types.item.get",
261-
"inventory-storage.service-points.collection.get",
262-
"inventory-storage.service-points.item.get",
263-
"inventory-storage.statistical-codes.collection.get",
264-
"inventory-storage.statistical-codes.item.get",
265-
"inventory.items.collection.get",
266-
"inventory.items.item.get",
267-
"okapi.proxy.tenants.modules.list",
268-
"proxiesfor.item.get",
269-
"user-settings.custom-fields.collection.get",
270-
"usergroups.collection.get",
271-
"usergroups.item.get",
272-
"users.collection.get",
273-
"users.item.get",
274-
"inventory-storage.instance-statuses.item.get",
275-
"inventory-storage.modes-of-issuance.item.get",
276-
"inventory-storage.instance-types.item.get",
277-
"inventory-storage.instance-types.collection.get",
278-
"inventory-storage.nature-of-content-terms.item.get",
279-
"inventory-storage.instance-formats.item.get",
280-
"inventory-storage.instance-note-types.item.get",
281-
"source-storage.source-records.collection.get",
282-
"user-tenants.collection.get",
283-
"consortium-search.holdings.collection.get",
284-
"consortium-search.holdings.batch.collection.get",
285-
"consortium-search.items.batch.collection.get",
286-
"consortium-search.items.collection.get",
287-
"consortia.user-tenants.collection.get",
288-
"perms.users.get",
289-
"permissions.users.item.get",
290-
"inventory-storage.statistical-code-types.item.get",
291-
"inventory-storage.subject-types.collection.get",
292-
"inventory-storage.subject-sources.collection.get",
293-
"inventory-storage.classification-types.collection.get"
294-
295-
]
296-
},
297-
{
298-
"methods": [ "POST" ],
299-
"pathPattern": "/bulk-edit/{id}/start",
300-
"permissionsRequired": [ "bulk-edit.start.item.post" ],
301-
"modulePermissions": [
302-
"users.collection.get",
303-
"users.item.get",
304-
"users.item.put",
305-
"usergroups.item.get",
306-
"addresstypes.item.get",
307-
"proxiesfor.item.get",
308-
"departments.item.get",
309-
"usergroups.collection.get",
310-
"addresstypes.collection.get",
311-
"inventory.items.item.get",
312-
"inventory.items.collection.get",
313-
"inventory.items.item.put",
314-
"inventory-storage.call-number-types.item.get",
315-
"inventory-storage.call-number-types.collection.get",
316-
"inventory-storage.item-damaged-statuses.item.get",
317-
"inventory-storage.item-damaged-statuses.collection.get",
318-
"inventory-storage.electronic-access-relationships.item.get",
319-
"inventory-storage.electronic-access-relationships.collection.get",
320-
"inventory-storage.item-note-types.item.get",
321-
"inventory-storage.item-note-types.collection.get",
322-
"inventory-storage.service-points.item.get",
323-
"inventory-storage.service-points.collection.get",
324-
"inventory-storage.statistical-codes.item.get",
325-
"inventory-storage.statistical-codes.collection.get",
326-
"inventory-storage.material-types.item.get",
327-
"inventory-storage.material-types.collection.get",
328-
"inventory-storage.loan-types.item.get",
329-
"inventory-storage.loan-types.collection.get",
330-
"inventory-storage.locations.item.get",
331-
"inventory-storage.locations.collection.get",
332-
"inventory-storage.instances.item.get",
333-
"inventory-storage.instances.collection.get",
334-
"inventory-storage.holdings.item.get",
335-
"inventory-storage.holdings.collection.get",
336-
"inventory-storage.holdings-types.collection.get",
337-
"inventory-storage.holdings-types.item.get",
338-
"inventory-storage.call-number-types.collection.get",
339-
"inventory-storage.call-number-types.item.get",
340-
"inventory-storage.holdings-note-types.collection.get",
341-
"inventory-storage.holdings-note-types.item.get",
342-
"inventory-storage.ill-policies.collection.get",
343-
"inventory-storage.ill-policies.item.get",
344-
"inventory-storage.holdings-sources.collection.get",
345-
"inventory-storage.holdings-sources.item.get",
346-
"inventory-storage.holdings.item.put",
347-
"perms.users.get",
348-
"permissions.users.item.get",
349-
"inventory-storage.statistical-code-types.item.get"
350-
]
351-
},
352-
{
353-
"methods": [ "GET" ],
354-
"pathPattern": "/bulk-edit/{jobId}/errors",
355-
"permissionsRequired": [ "bulk-edit.errors.collection.get" ],
356-
"modulePermissions": []
357-
},
358225
{
359226
"methods": [ "GET" ],
360227
"pathPattern": "/refresh-presigned-url",
@@ -412,36 +279,11 @@
412279
}
413280
],
414281
"permissionSets": [
415-
{
416-
"permissionName" : "bulk-edit.item.post",
417-
"displayName" : "upload identifiers list",
418-
"description" : "Upload list of item identifiers"
419-
},
420-
{
421-
"permissionName": "bulk-edit.start.item.post",
422-
"displayName": "start update items",
423-
"description": "Start update items"
424-
},
425-
{
426-
"permissionName" : "bulk-edit.errors.collection.get",
427-
"displayName" : "get errors for preview",
428-
"description" : "Get errors for preview"
429-
},
430282
{
431283
"permissionName" : "refresh-presigned-url.get",
432284
"displayName" : "Get refreshed presigned url for export file",
433285
"description" : "Get refreshed presigned url for export file"
434286
},
435-
{
436-
"permissionName" : "bulk-edit.all",
437-
"displayName" : "bulk-edit all",
438-
"description" : "All permissions for bulk-edit module",
439-
"subPermissions" : [
440-
"bulk-edit.item.post",
441-
"bulk-edit.start.item.post",
442-
"bulk-edit.errors.collection.get"
443-
]
444-
},
445287
{
446288
"permissionName" : "refresh-presigned-url.all",
447289
"displayName" : "refresh-presigned-url all",

‎pom.xml‎

Lines changed: 24 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,6 @@
3535
<bursar-export.yaml.file>
3636
${project.basedir}/src/main/resources/swagger.api/bursar-export.yaml
3737
</bursar-export.yaml.file>
38-
<bulk-edit.yaml.file>
39-
${project.basedir}/src/main/resources/swagger.api/bulk-edit.yaml
40-
</bulk-edit.yaml.file>
4138
<order-export.yaml.file>
4239
${project.basedir}/src/main/resources/swagger.api/order-export.yaml
4340
</order-export.yaml.file>
@@ -89,8 +86,6 @@
8986
**/src/main/java/org/folio/de/entity/**
9087
</sonar.exclusions>
9188

92-
<sonar.coverage.exclusions>**/src/main/java/org/folio/dew/service/BulkEditParseService.java</sonar.coverage.exclusions>
93-
9489
<argLine />
9590
</properties>
9691

@@ -539,56 +534,30 @@
539534
</configuration>
540535
</execution>
541536
<execution>
542-
<id>bulk-edit-worker</id>
543-
<goals>
544-
<goal>generate</goal>
545-
</goals>
546-
<configuration>
547-
<inputSpec>${bulk-edit.yaml.file}</inputSpec>
548-
<output>${project.build.directory}/generated-sources</output>
549-
<generatorName>spring</generatorName>
550-
<modelPackage>${project.groupId}.dew.domain.dto</modelPackage>
551-
<generateApis>true</generateApis>
552-
<generateApiTests>false</generateApiTests>
553-
<generateApiDocumentation>false</generateApiDocumentation>
554-
<generateModels>true</generateModels>
555-
<generateModelTests>false</generateModelTests>
556-
<generateSupportingFiles>true</generateSupportingFiles>
557-
<generateModelDocumentation>true</generateModelDocumentation>
558-
<configOptions>
559-
<java8>true</java8>
560-
<dateLibrary>java</dateLibrary>
561-
<interfaceOnly>true</interfaceOnly>
562-
<useSpringBoot3>true</useSpringBoot3>
563-
<serializableModel>true</serializableModel>
564-
</configOptions>
565-
</configuration>
566-
</execution>
567-
<execution>
568-
<id>refresh-presigned-url</id>
569-
<goals>
570-
<goal>generate</goal>
571-
</goals>
572-
<configuration>
573-
<inputSpec>${refresh-presigned-url.yaml.file}</inputSpec>
574-
<output>${project.build.directory}/generated-sources</output>
575-
<generatorName>spring</generatorName>
576-
<modelPackage>${project.groupId}.dew.domain.dto</modelPackage>
577-
<generateApis>true</generateApis>
578-
<generateApiTests>false</generateApiTests>
579-
<generateApiDocumentation>true</generateApiDocumentation>
580-
<generateModels>true</generateModels>
581-
<generateModelTests>false</generateModelTests>
582-
<generateSupportingFiles>true</generateSupportingFiles>
583-
<generateModelDocumentation>true</generateModelDocumentation>
584-
<supportingFilesToGenerate>ApiUtil.java</supportingFilesToGenerate>
585-
<configOptions>
586-
<java8>true</java8>
587-
<dateLibrary>java</dateLibrary>
588-
<interfaceOnly>true</interfaceOnly>
589-
<useSpringBoot3>true</useSpringBoot3>
590-
</configOptions>
591-
</configuration>
537+
<id>refresh-presigned-url</id>
538+
<goals>
539+
<goal>generate</goal>
540+
</goals>
541+
<configuration>
542+
<inputSpec>${refresh-presigned-url.yaml.file}</inputSpec>
543+
<output>${project.build.directory}/generated-sources</output>
544+
<generatorName>spring</generatorName>
545+
<modelPackage>${project.groupId}.dew.domain.dto</modelPackage>
546+
<generateApis>true</generateApis>
547+
<generateApiTests>false</generateApiTests>
548+
<generateApiDocumentation>true</generateApiDocumentation>
549+
<generateModels>true</generateModels>
550+
<generateModelTests>false</generateModelTests>
551+
<generateSupportingFiles>true</generateSupportingFiles>
552+
<generateModelDocumentation>true</generateModelDocumentation>
553+
<supportingFilesToGenerate>ApiUtil.java</supportingFilesToGenerate>
554+
<configOptions>
555+
<java8>true</java8>
556+
<dateLibrary>java</dateLibrary>
557+
<interfaceOnly>true</interfaceOnly>
558+
<useSpringBoot3>true</useSpringBoot3>
559+
</configOptions>
560+
</configuration>
592561
</execution>
593562
<execution>
594563
<id>order-export-worker</id>

0 commit comments

Comments
 (0)