Skip to content

Commit f37b559

Browse files
authored
BHBC-2140: Add functions for safely lowercasing/trimming unknown values (#939)
1 parent fee0b44 commit f37b559

9 files changed

Lines changed: 235 additions & 60 deletions

File tree

api/src/utils/media/csv/csv-file.ts

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import xlsx from 'xlsx';
22
import { SUBMISSION_MESSAGE_TYPE } from '../../../constants/status';
3+
import { safeToLowerCase, safeTrim } from '../../string-utils';
34
import { IMediaState, MediaValidation } from '../media-file';
45
import { getCellValue, getWorksheetRange, replaceCellDates, trimCellWhitespace } from '../xlsx/xlsx-utils';
56

@@ -101,7 +102,7 @@ export class CSVWorksheet {
101102

102103
if (aoaHeaders.length > 0) {
103104
// Parse the headers array from the array of arrays produced by calling `xlsx.utils.sheet_to_json`
104-
this._headers = aoaHeaders[0].map((item) => item?.trim());
105+
this._headers = aoaHeaders[0].map(safeTrim);
105106
}
106107
}
107108

@@ -110,7 +111,7 @@ export class CSVWorksheet {
110111

111112
getHeadersLowerCase(): string[] {
112113
if (!this._headersLowerCase.length) {
113-
this._headersLowerCase = this.getHeaders().map((item) => item?.toLowerCase());
114+
this._headersLowerCase = this.getHeaders().map(safeToLowerCase);
114115
}
115116

116117
return this._headersLowerCase;

api/src/utils/media/csv/validation/csv-header-validator.ts

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { SUBMISSION_MESSAGE_TYPE } from '../../../../constants/status';
2+
import { safeToLowerCase, safeTrim } from '../../../string-utils';
23
import { CSVValidator } from '../csv-file';
34

45
/**
@@ -62,7 +63,7 @@ export const hasRequiredHeadersValidator = (config?: FileRequiredHeaderValidator
6263
const headersLowerCase = csvWorksheet.getHeadersLowerCase();
6364

6465
for (const requiredHeader of config.file_required_columns_validator.required_columns) {
65-
if (!headersLowerCase.includes(requiredHeader.toLowerCase())) {
66+
if (!headersLowerCase.includes(safeToLowerCase(requiredHeader))) {
6667
csvWorksheet.csvValidation.addHeaderErrors([
6768
{
6869
errorCode: SUBMISSION_MESSAGE_TYPE.MISSING_REQUIRED_HEADER,
@@ -118,7 +119,7 @@ export const hasRecommendedHeadersValidator = (config?: FileRecommendedHeaderVal
118119
}
119120

120121
for (const recommendedHeader of config.file_recommended_columns_validator.recommended_columns) {
121-
if (!headersLowerCase.includes(recommendedHeader.toLowerCase())) {
122+
if (!headersLowerCase.includes(safeToLowerCase(recommendedHeader))) {
122123
csvWorksheet.csvValidation.addHeaderWarnings([
123124
{
124125
errorCode: SUBMISSION_MESSAGE_TYPE.MISSING_RECOMMENDED_HEADER,
@@ -162,8 +163,8 @@ export const getValidHeadersValidator = (config?: FileValidHeadersValidatorConfi
162163
for (const header of headers) {
163164
if (
164165
!config.file_valid_columns_validator.valid_columns
165-
.map((item) => item.toLowerCase())
166-
.includes(header.trim().toLowerCase())
166+
.map(safeToLowerCase)
167+
.includes(safeToLowerCase(safeTrim(header)))
167168
) {
168169
csvWorksheet.csvValidation.addHeaderWarnings([
169170
{

api/src/utils/media/csv/validation/csv-row-validator.ts

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { SUBMISSION_MESSAGE_TYPE } from '../../../../constants/status';
2+
import { safeToLowerCase } from '../../../string-utils';
23
import { CSVValidator } from '../csv-file';
34

45
export type RequiredFieldsValidatorConfig = {
@@ -21,7 +22,7 @@ export const getRequiredFieldsValidator = (config?: RequiredFieldsValidatorConfi
2122
const headersLowerCase = csvWorksheet.getHeadersLowerCase();
2223

2324
rows.forEach((row, rowIndex) => {
24-
const columnIndex = headersLowerCase.indexOf(config.columnName.toLowerCase());
25+
const columnIndex = headersLowerCase.indexOf(safeToLowerCase(config.columnName));
2526

2627
// if column does not exist, return
2728
if (columnIndex < 0) {
@@ -80,7 +81,7 @@ export const getCodeValueFieldsValidator = (config?: ColumnCodeValidatorConfig):
8081
const headersLowerCase = csvWorksheet.getHeadersLowerCase();
8182

8283
rows.forEach((row, rowIndex) => {
83-
const columnIndex = headersLowerCase.indexOf(config.columnName.toLowerCase());
84+
const columnIndex = headersLowerCase.indexOf(safeToLowerCase(config.columnName));
8485

8586
// if column does not exist, return
8687
if (columnIndex < 0) {
@@ -95,14 +96,14 @@ export const getCodeValueFieldsValidator = (config?: ColumnCodeValidatorConfig):
9596
}
9697

9798
// compare allowed code values as lowercase strings
98-
const allowedCodeValuesLowerCase: string[] = [];
99+
const allowedCodeValuesLowerCase: (string | number)[] = [];
99100
const allowedCodeValues = config.column_code_validator.allowed_code_values.map((allowedCode) => {
100-
allowedCodeValuesLowerCase.push(allowedCode.name?.toString().toLowerCase());
101+
allowedCodeValuesLowerCase.push(safeToLowerCase(allowedCode.name));
101102
return allowedCode.name;
102103
});
103104

104105
// Add an error if the cell value is not one of the elements in the codeValues array
105-
if (!allowedCodeValuesLowerCase.includes(rowValueForColumn?.toLowerCase())) {
106+
if (!allowedCodeValuesLowerCase.includes(safeToLowerCase(rowValueForColumn))) {
106107
csvWorksheet.csvValidation.addRowErrors([
107108
{
108109
errorCode: SUBMISSION_MESSAGE_TYPE.INVALID_VALUE,
@@ -147,7 +148,7 @@ export const getValidRangeFieldsValidator = (config?: ColumnRangeValidatorConfig
147148
const headersLowerCase = csvWorksheet.getHeadersLowerCase();
148149

149150
rows.forEach((row, rowIndex) => {
150-
const columnIndex = headersLowerCase.indexOf(config.columnName.toLowerCase());
151+
const columnIndex = headersLowerCase.indexOf(safeToLowerCase(config.columnName));
151152

152153
// if column does not exist, return
153154
if (columnIndex < 0) {
@@ -248,7 +249,7 @@ export const getNumericFieldsValidator = (config?: ColumnNumericValidatorConfig)
248249
const headersLowerCase = csvWorksheet.getHeadersLowerCase();
249250

250251
rows.forEach((row, rowIndex) => {
251-
const columnIndex = headersLowerCase.indexOf(config.columnName.toLowerCase());
252+
const columnIndex = headersLowerCase.indexOf(safeToLowerCase(config.columnName));
252253

253254
// if column does not exist, return
254255
if (columnIndex < 0) {
@@ -311,7 +312,7 @@ export const getValidFormatFieldsValidator = (config?: ColumnFormatValidatorConf
311312
const headersLowerCase = csvWorksheet.getHeadersLowerCase();
312313

313314
rows.forEach((row, rowIndex) => {
314-
const columnIndex = headersLowerCase.indexOf(config.columnName.toLowerCase());
315+
const columnIndex = headersLowerCase.indexOf(safeToLowerCase(config.columnName));
315316

316317
// if column does not exist, return
317318
if (columnIndex < 0) {
@@ -367,7 +368,7 @@ export const getUniqueColumnsValidator = (config?: FileColumnUniqueValidatorConf
367368

368369
// find the indices of all provided column names in the worksheet
369370
const columnIndices = config.file_column_unique_validator.column_names.map((column) =>
370-
lowercaseHeaders.indexOf(column.toLocaleLowerCase())
371+
lowercaseHeaders.indexOf(safeToLowerCase(column))
371372
);
372373

373374
// checks list of column indices if any are missing (-1) and returns early
@@ -377,7 +378,7 @@ export const getUniqueColumnsValidator = (config?: FileColumnUniqueValidatorConf
377378

378379
rows.forEach((row, rowIndex) => {
379380
const key = config.file_column_unique_validator.column_names
380-
.map((columnIndex) => `${row[columnIndex] || ''}`.trim().toLocaleLowerCase())
381+
.map((columnIndex) => `${row[columnIndex] || ''}`.trim().toLowerCase())
381382
.join(', ');
382383
// check if key exists already
383384
if (!keySet.has(key)) {

api/src/utils/media/validation/file-type-and-content-validator.ts

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import { safeToLowerCase } from '../../string-utils';
12
import { DWCArchive, DWCArchiveValidator } from '../dwc/dwc-archive-file';
23
import { MediaValidator } from '../media-file';
34
import { XLSXCSV, XLSXCSVValidator } from '../xlsx/xlsx-file';
@@ -93,7 +94,7 @@ const checkRequiredFieldsInDWCArchive = (dwcArchive: DWCArchive, config: Submiss
9394
const fileNames = dwcArchive.rawFile.mediaFiles.map((mediaFile) => mediaFile.name);
9495

9596
config.submission_required_files_validator.required_files.forEach((requiredFile) => {
96-
if (!fileNames.includes(requiredFile.toLowerCase())) {
97+
if (!fileNames.includes(safeToLowerCase(requiredFile))) {
9798
dwcArchive.mediaValidation.addFileErrors([`Missing required file: ${requiredFile}`]);
9899
}
99100
});
@@ -112,10 +113,10 @@ const checkRequiredFieldsInXLSXCSV = (xlsxCsv: XLSXCSV, config: SubmissionRequir
112113
return xlsxCsv;
113114
}
114115

115-
const worksheetNames = Object.keys(xlsxCsv.workbook.worksheets).map((item) => item.toLowerCase());
116+
const worksheetNames = Object.keys(xlsxCsv.workbook.worksheets).map(safeToLowerCase);
116117

117118
config.submission_required_files_validator.required_files.forEach((requiredFile) => {
118-
if (!worksheetNames.includes(requiredFile.toLowerCase())) {
119+
if (!worksheetNames.includes(safeToLowerCase(requiredFile))) {
119120
xlsxCsv.mediaValidation.addFileErrors([`Missing required sheet: ${requiredFile}`]);
120121
}
121122
});

api/src/utils/media/xlsx/validation/xlsx-validation.ts

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { SUBMISSION_MESSAGE_TYPE } from '../../../../constants/status';
2+
import { safeTrim } from '../../../string-utils';
23
import { CSVWorkBook, WorkBookValidator } from '../../csv/csv-file';
34

45
export type ParentChildKeyMatchValidatorConfig = {
@@ -48,7 +49,7 @@ export const getParentChildKeyMatchValidator = (config?: ParentChildKeyMatchVali
4849
}
4950

5051
// Filter column names to only check key violation on columns included in the child sheet
51-
const filteredColumnNames = column_names.filter((columnName: string) => Boolean(childRowObjects[0][columnName]));
52+
const filteredColumnNames = column_names.filter((columnName) => Boolean(childRowObjects[0][columnName]));
5253

5354
/**
5455
* Encodes the column values for a worksheet at a given row into a string, which is used for comparison with another worksheet
@@ -65,7 +66,7 @@ export const getParentChildKeyMatchValidator = (config?: ParentChildKeyMatchVali
6566
.filter(Boolean)
6667

6768
// Trim whitespace
68-
.map((columnValue: string) => columnValue.trim())
69+
.map(safeTrim)
6970

7071
// Deliminate column values
7172
.join('|')

api/src/utils/media/xlsx/xlsx-utils.ts

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import xlsx, { CellObject } from 'xlsx';
2+
import { safeTrim } from '../../string-utils';
23

34
/**
45
* Get a worksheet by name.
@@ -63,12 +64,12 @@ export function prepareWorksheetCells(worksheet: xlsx.WorkSheet) {
6364
export function trimCellWhitespace(cell: CellObject) {
6465
// check and clean raw strings
6566
if (cell.t === 's') {
66-
cell.v = (cell.v as string).trim();
67+
cell.v = safeTrim(cell.v);
6768
}
6869

6970
// check and clean formatted strings
7071
if (cell.w) {
71-
cell.w = cell.w.trim();
72+
cell.w = safeTrim(cell.w);
7273
}
7374

7475
return cell;

api/src/utils/string-utils.test.ts

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
import { expect } from 'chai';
2+
import { safeToLowerCase, safeTrim } from './string-utils';
3+
4+
describe('safeToLowerCase', () => {
5+
describe('returns value lowercase', () => {
6+
it('when value is a lowercase string', () => {
7+
expect(safeToLowerCase('string')).to.equal('string');
8+
});
9+
10+
it('when value is an uppercase string', () => {
11+
expect(safeToLowerCase('STRING')).to.equal('string');
12+
});
13+
14+
it('when value is a mixed case string', () => {
15+
expect(safeToLowerCase('sTRiNG')).to.equal('string');
16+
});
17+
});
18+
19+
describe('returns value unaltered', () => {
20+
it('when value is a negative number', () => {
21+
expect(safeToLowerCase(-123)).to.equal(-123);
22+
});
23+
24+
it('when value is a zero', () => {
25+
expect(safeToLowerCase(0)).to.equal(0);
26+
});
27+
28+
it('when value is a positive number', () => {
29+
expect(safeToLowerCase(123)).to.equal(123);
30+
});
31+
32+
it('when value is `false`', () => {
33+
expect(safeToLowerCase(false)).to.equal(false);
34+
});
35+
36+
it('when value is `true`', () => {
37+
expect(safeToLowerCase(true)).to.equal(true);
38+
});
39+
40+
it('when value is an empty object', () => {
41+
expect(safeToLowerCase({})).to.eql({});
42+
});
43+
44+
it('when value is an empty array', () => {
45+
expect(safeToLowerCase([])).to.eql([]);
46+
});
47+
48+
it('when value is a non-empty array of numbers', () => {
49+
expect(safeToLowerCase([1, 2, 3])).to.eql([1, 2, 3]);
50+
});
51+
52+
it('when value is a non-empty array of strings', () => {
53+
expect(safeToLowerCase(['1', 'string', 'false'])).to.eql(['1', 'string', 'false']);
54+
});
55+
56+
it('when value is a function', () => {
57+
const fn = (a: number, b: number) => a * b;
58+
expect(safeToLowerCase(fn)).to.equal(fn);
59+
});
60+
});
61+
});
62+
63+
describe('safeTrim', () => {
64+
describe('returns value trimmed', () => {
65+
it('when value is a lowercase string', () => {
66+
expect(safeTrim(' string ')).to.equal('string');
67+
});
68+
69+
it('when value is an uppercase string', () => {
70+
expect(safeTrim(' STRING ')).to.equal('STRING');
71+
});
72+
73+
it('when value is a mixed case string', () => {
74+
expect(safeTrim(' sTRiNG ')).to.equal('sTRiNG');
75+
});
76+
});
77+
78+
describe('returns value unaltered', () => {
79+
it('when value is a negative number', () => {
80+
expect(safeTrim(-123)).to.equal(-123);
81+
});
82+
83+
it('when value is a zero', () => {
84+
expect(safeTrim(0)).to.equal(0);
85+
});
86+
87+
it('when value is a positive number', () => {
88+
expect(safeTrim(123)).to.equal(123);
89+
});
90+
91+
it('when value is `false`', () => {
92+
expect(safeTrim(false)).to.equal(false);
93+
});
94+
95+
it('when value is `true`', () => {
96+
expect(safeTrim(true)).to.equal(true);
97+
});
98+
99+
it('when value is an empty object', () => {
100+
expect(safeTrim({})).to.eql({});
101+
});
102+
103+
it('when value is an empty array', () => {
104+
expect(safeTrim([])).to.eql([]);
105+
});
106+
107+
it('when value is a non-empty array of numbers', () => {
108+
expect(safeTrim([1, 2, 3])).to.eql([1, 2, 3]);
109+
});
110+
111+
it('when value is a non-empty array of strings', () => {
112+
expect(safeTrim([' 1 ', ' string ', ' false '])).to.eql([' 1 ', ' string ', ' false ']);
113+
});
114+
115+
it('when value is a function', () => {
116+
const fn = (a: number, b: number) => a * b;
117+
expect(safeTrim(fn)).to.equal(fn);
118+
});
119+
});
120+
});

api/src/utils/string-utils.ts

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
import { isString } from 'lodash';
2+
3+
/**
4+
* Safely apply `.toLowerCase()` to a value of unknown type.
5+
*
6+
* If the value is not a string, then the original unaltered value will be returned.
7+
*
8+
* @export
9+
* @template T
10+
* @param {T} value
11+
* @return {*} {T}
12+
*/
13+
export function safeToLowerCase<T>(value: T): T {
14+
if (isString(value)) {
15+
return (value.toLowerCase() as unknown) as T;
16+
}
17+
18+
return value;
19+
}
20+
21+
/**
22+
* Safely apply `.trim()` to a value of unknown type.
23+
*
24+
* If the value is not a string, then the original unaltered value will be returned.
25+
*
26+
* @export
27+
* @template T
28+
* @param {T} value
29+
* @return {*} {T}
30+
*/
31+
export function safeTrim<T>(value: T): T {
32+
if (isString(value)) {
33+
return (value.trim() as unknown) as T;
34+
}
35+
36+
return value;
37+
}

0 commit comments

Comments
 (0)