Skip to content

Commit dfdd598

Browse files
authored
Merge pull request #73 from bcgov/feature/refactor-meta-service
Separate Metadata and Metadada joins in service
2 parents 2e39ec6 + 61d69b2 commit dfdd598

File tree

9 files changed

+168
-72
lines changed

9 files changed

+168
-72
lines changed

app/src/components/utils.js

Lines changed: 9 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -21,17 +21,6 @@ const utils = {
2121
else return str;
2222
},
2323

24-
/**
25-
* @function toLowerCaseKeys
26-
* @param {object} obj an object with keys of mixed-case
27-
* @returns {object} the input object with keys converted to lowercase
28-
*/
29-
toLowerCaseKeys(obj){
30-
return Object.fromEntries(Object.entries(obj).map(entry => {
31-
return [entry[0].toLowerCase(), entry[1]];
32-
}));
33-
},
34-
3524
/**
3625
* @function delimit
3726
* Yields a string `s` that will always have a trailing delimiter. Returns an empty string if falsy.
@@ -134,7 +123,7 @@ const utils = {
134123
getMetadata(obj) {
135124
return Object.fromEntries(Object.keys(obj)
136125
.filter((key) => key.toLowerCase().startsWith('x-amz-meta-'))
137-
.map((key) => ([key.substring(11), obj[key]]))
126+
.map((key) => ([key.toLowerCase().substring(11), obj[key]]))
138127
);
139128
},
140129

@@ -196,15 +185,15 @@ const utils = {
196185
},
197186

198187
/**
199-
* @function getTagsByKeyValue
200-
* get tag objects in array that have given key and value
201-
* @param {object[]} tags and array of tags (eg: [{ key: 'a', value: '1'}, { key: 'b', value: '1'}]
202-
* @param {string} key the string to match in the tag's `key` property
203-
* @param {string} value the string to match in the tag's `value` property
204-
* @returns {object[]} an array of matching tag objects
188+
* @function getObjectsByKeyValue
189+
* get tag/metadata objects in array that have given key and value
190+
* @param {object[]} array and array of objects (eg: [{ key: 'a', value: '1'}, { key: 'b', value: '1'}]
191+
* @param {string} key the string to match in the objects's `key` property
192+
* @param {string} value the string to match in the objects's `value` property
193+
* @returns {object[]} an array of matching objects
205194
*/
206-
getTagsByKeyValue(tags, key, value){
207-
return tags.find(tag => (tag.key === key && tag.value === value));
195+
getObjectsByKeyValue(array, key, value){
196+
return array.find(obj => (obj.key === key && obj.value === value));
208197
},
209198

210199
/**

app/src/controllers/object.js

Lines changed: 11 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ const {
1313
addDashesToUuid,
1414
getAppAuthMode,
1515
getKeyValue,
16-
toLowerCaseKeys,
16+
toLowerKeys,
1717
getMetadata,
1818
getPath,
1919
isTruthy,
@@ -130,7 +130,7 @@ const controller = {
130130
await versionService.update({ ...data, id: objId }, userId, trx);
131131

132132
// update metadata for version in DB
133-
await metadataService.updateMetadata(version.id, data.metadata, userId, trx);
133+
await metadataService.associateMetadata(version.id, getKeyValue(data.metadata), userId, trx);
134134
});
135135

136136
res.status(204).end();
@@ -177,8 +177,8 @@ const controller = {
177177
// Add tags to version in DB
178178
await utils.trxWrapper(async (trx) => {
179179
const version = await versionService.get(data.versionId, objId, trx);
180-
// callling replaceTags() in case they are replacing an existing tag with same key which we need to dissociate
181-
await tagService.replaceTags(version.id, data.tags.map(tag => toLowerCaseKeys(tag)), userId, trx);
180+
// use replaceTags() in case they are replacing an existing tag with same key which we need to dissociate
181+
await tagService.replaceTags(version.id, toLowerKeys(data.tags), userId, trx);
182182
});
183183

184184
res.status(204).end();
@@ -230,7 +230,7 @@ const controller = {
230230
const versions = await versionService.create(data, userId, trx);
231231

232232
// add metadata to version in DB
233-
if (data.metadata && Object.keys(data.metadata).length) await metadataService.updateMetadata(versions.id, data.metadata, userId, trx);
233+
await metadataService.associateMetadata(versions.id, getKeyValue(data.metadata), userId, trx);
234234

235235
// add tags to version in DB
236236
if (data.tags && Object.keys(data.tags).length) await tagService.associateTags(versions.id, getKeyValue(data.tags), userId, trx);
@@ -318,9 +318,8 @@ const controller = {
318318
const version = s3Response.VersionId ?
319319
await versionService.copy(sourceVersionId, s3Response.VersionId, objId, userId, trx) :
320320
await versionService.update({ ...data, id: objId }, userId, trx);
321-
322321
// add metadata to version in DB
323-
await metadataService.updateMetadata(version.id, data.metadata, userId, trx);
322+
await metadataService.associateMetadata(version.id, getKeyValue(data.metadata), userId, trx);
324323
});
325324

326325
res.status(204).end();
@@ -423,8 +422,8 @@ const controller = {
423422
let dissociateTags = [];
424423
if (req.query.tagset) {
425424
dissociateTags = getKeyValue(req.query.tagset);
426-
} else if (objectTagging.TagSet) {
427-
dissociateTags = objectTagging.TagSet.map(tag => toLowerCaseKeys(tag));
425+
} else if (objectTagging.TagSet && objectTagging.TagSet.length) {
426+
dissociateTags = toLowerKeys(objectTagging.TagSet);
428427
}
429428
if (dissociateTags.length) await tagService.dissociateTags(version.id, dissociateTags, userId, trx);
430429
});
@@ -591,7 +590,7 @@ const controller = {
591590
await versionService.update({ ...data, id: objId }, userId, trx);
592591

593592
// add metadata
594-
await metadataService.updateMetadata(version.id, data.metadata, userId, trx);
593+
await metadataService.associateMetadata(version.id, getKeyValue(data.metadata), userId, trx);
595594
});
596595

597596
res.status(204).end();
@@ -634,7 +633,7 @@ const controller = {
634633
// update tags on version in DB
635634
await utils.trxWrapper(async (trx) => {
636635
const version = await versionService.get(data.versionId, objId, trx);
637-
await tagService.replaceTags(version.id, data.tags.map(tag => toLowerCaseKeys(tag)), userId, trx);
636+
await tagService.replaceTags(version.id, toLowerKeys(data.tags), userId, trx);
638637
});
639638

640639
res.status(204).end();
@@ -759,7 +758,7 @@ const controller = {
759758
}
760759

761760
// add metadata to version in DB
762-
if (data.metadata && Object.keys(data.metadata).length) await metadataService.updateMetadata(version.id, data.metadata, userId, trx);
761+
if (data.metadata && Object.keys(data.metadata).length) await metadataService.associateMetadata(version.id, getKeyValue(data.metadata), userId, trx);
763762

764763
// add tags to version in DB
765764
if (data.tags && Object.keys(data.tags).length) await tagService.replaceTags(version.id, getKeyValue(data.tags), userId, trx);

app/src/db/models/tables/metadata.js

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ class Metadata extends Model {
77

88
static get relationMappings() {
99
const Version = require('./version');
10+
const VersionMetadata = require('./versionMetadata');
1011

1112
return {
1213
version: {
@@ -20,7 +21,15 @@ class Metadata extends Model {
2021
},
2122
to: 'version.id'
2223
}
23-
}
24+
},
25+
versionMetadata: {
26+
relation: Model.HasManyRelation,
27+
modelClass: VersionMetadata,
28+
join: {
29+
from: 'metadata.id',
30+
to: 'version_metadata.metadataId'
31+
}
32+
},
2433
};
2534
}
2635

app/src/docs/v1.api-spec.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -778,6 +778,7 @@ components:
778778
description: >-
779779
An arbitrary metadata key/value pair. Must contain the x-amz-meta-
780780
prefix to be valid. Multiple metadata pairs can be defined.
781+
keys must be unique and will be converted to lowercase.
781782
schema:
782783
type: string
783784
example:

app/src/services/metadata.js

Lines changed: 122 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,53 +1,145 @@
11
const { NIL: SYSTEM_USER } = require('uuid');
22
const { Metadata, VersionMetadata } = require('../db/models');
3-
const { getKeyValue } = require('../components/utils');
3+
const { getObjectsByKeyValue } = require('../components/utils');
44

55
/**
66
* The Metadata DB Service
77
*/
88
const service = {
99

10+
1011
/**
11-
* @function updateMetadata
12-
* Updates metadata and relates them to the associated version
13-
* Un-relates any existing metadata for this version
12+
* @function associateMetadata
13+
* Makes the incoming list of metadata the definitive set associated with versionId
14+
* Dissociaate extraneous metadata and also does collision detection for null versions (non-versioned)
1415
* @param {string} versionId The uuid id column from version table
15-
* @param {object} metadata Incoming object with `<key>:<value>` metadata to add for this version
16+
* @param {object[]} metadata Incoming array of metadata objects to add for this version (eg: [{ key: 'a', value: '1'}, {key: 'B', value: '2'}]).
17+
* This will always be the difinitive metadata we want on the version
1618
* @param {string} [currentUserId=SYSTEM_USER] The optional userId uuid actor; defaults to system user if unspecified
1719
* @param {object} [etrx=undefined] An optional Objection Transaction object
1820
* @returns {Promise<object>} The result of running the insert operation
1921
* @throws The error encountered upon db transaction failure
2022
*/
21-
updateMetadata: async (versionId, metadata, currentUserId = SYSTEM_USER, etrx = undefined) => {
23+
associateMetadata: async (versionId, metadata, currentUserId = SYSTEM_USER, etrx = undefined) => {
2224
let trx;
2325
try {
2426
trx = etrx ? etrx : await Metadata.startTransaction();
2527
let response = [];
2628

27-
// convert metadata to array for DB insert query
28-
const arr = getKeyValue(metadata);
29-
if (arr.length) {
30-
// insert/merge metadata records
31-
const insertMetadata = await Metadata.query(trx)
32-
.insert(arr)
33-
.onConflict(['key', 'value'])
34-
.merge(); // required to include id's of existing rows in result
35-
36-
// un-relate all existing version_metadata
37-
// this only happens when updating the single 'null version' record in db, when using a non-versioned bucket
38-
// otherwise joining records remain in db for previous versions
39-
await VersionMetadata.query(trx)
40-
.delete()
41-
.where('versionId', versionId);
42-
43-
// add new version_metadata records
44-
const relateArray = insertMetadata.map(({id}) => ({
45-
versionId: versionId,
46-
metadataId: id,
47-
createdBy: currentUserId
48-
}));
49-
response = await VersionMetadata.query(trx)
50-
.insert(relateArray);
29+
if (metadata && metadata.length) {
30+
// get DB records of all input metadata
31+
const dbMetadata = await service.createMetadata(metadata, trx);
32+
33+
// already joined
34+
const associatedMetadata = await VersionMetadata.query(trx)
35+
.modify('filterVersionId', versionId);
36+
37+
// remove existing joins for metadata that is not in incomming set
38+
if (associatedMetadata.length) {
39+
const dissociateMetadata = associatedMetadata.filter(({ metadataId }) => !dbMetadata.some(({ id }) => id === metadataId));
40+
if (dissociateMetadata.length) {
41+
await VersionMetadata.query(trx)
42+
.whereIn('id', dissociateMetadata.map(meta => meta.id))
43+
.modify('filterVersionId', versionId)
44+
.delete();
45+
46+
// delete all orphaned metadata records
47+
await service.pruneOrphanedMetadata(trx);
48+
}
49+
}
50+
51+
// join new metadata
52+
const newJoins = associatedMetadata.length ? dbMetadata.filter(({ id }) => !associatedMetadata.some(({ metadataId }) => metadataId === id)) : dbMetadata;
53+
54+
if (newJoins.length) {
55+
response = await VersionMetadata.query(trx)
56+
.insert(newJoins.map(({ id }) => ({
57+
versionId: versionId,
58+
metadataId: id,
59+
createdBy: currentUserId
60+
})));
61+
}
62+
}
63+
64+
if (!etrx) await trx.commit();
65+
return Promise.resolve(response);
66+
} catch (err) {
67+
if (!etrx && trx) await trx.rollback();
68+
throw err;
69+
}
70+
},
71+
72+
/**
73+
* @function deleteOrphanedMetadata
74+
* deletes Metadata records if they are no longer related to any versions
75+
* @param {object} [etrx=undefined] An optional Objection Transaction object
76+
* @returns {Promise<number>} The result of running the delete operation (number of rows deleted)
77+
* @throws The error encountered upon db transaction failure
78+
*/
79+
// TODO: check if deleteing a version will prune orphan metadata records (sister table)
80+
pruneOrphanedMetadata: async (etrx = undefined) => {
81+
let trx;
82+
try {
83+
trx = etrx ? etrx : await Metadata.startTransaction();
84+
85+
const deletedMetadataIds = await Metadata.query(trx)
86+
.allowGraph('versionMetadata')
87+
.withGraphJoined('versionMetadata')
88+
.select('metadata.id')
89+
.whereNull('versionMetadata.metadataId');
90+
91+
const response = await Metadata.query(trx)
92+
.delete()
93+
.whereIn('id', deletedMetadataIds.map(({ id }) => id));
94+
95+
if (!etrx) await trx.commit();
96+
return Promise.resolve(response);
97+
} catch (err) {
98+
if (!etrx && trx) await trx.rollback();
99+
throw err;
100+
}
101+
},
102+
103+
/**
104+
* @function createMetadata
105+
* Inserts any metadata records if they dont already exist in db
106+
* @param {object} metadata Incoming object with `<key>:<value>` metadata to add for this version
107+
* @param {object} [etrx=undefined] An optional Objection Transaction object
108+
* @returns {Promise<object>} an array of all input metadata
109+
* @throws The error encountered upon db transaction failure
110+
*/
111+
createMetadata: async (metadata, etrx = undefined) => {
112+
let trx;
113+
let response = [];
114+
try {
115+
trx = etrx ? etrx : await Metadata.startTransaction();
116+
117+
// get all metadata already in db
118+
const allMetadata = await Metadata.query(trx).select();
119+
const existingMetadata = [];
120+
const newMetadata = [];
121+
122+
metadata.forEach(({ key, value }) => {
123+
// if metadata is already in db
124+
if (getObjectsByKeyValue(allMetadata, key, value)) {
125+
// add metadata object to existingMetadata array
126+
existingMetadata.push({ id: getObjectsByKeyValue(allMetadata, key, value).id, key: key, value: value });
127+
}
128+
// else add to array for inserting
129+
else {
130+
newMetadata.push({ key: key, value: value });
131+
}
132+
});
133+
// insert new metadata
134+
if (newMetadata.length) {
135+
const newMetadataRecords = await Metadata.query(trx)
136+
.insert(newMetadata)
137+
.returning('*');
138+
// merge new with existing metadata
139+
response = existingMetadata.concat(newMetadataRecords);
140+
}
141+
else {
142+
response = existingMetadata;
51143
}
52144

53145
if (!etrx) await trx.commit();

app/src/services/tag.js

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
const { NIL: SYSTEM_USER } = require('uuid');
22
const { Tag, VersionTag } = require('../db/models');
3-
const { getTagsByKeyValue } = require('../components/utils');
3+
const { getObjectsByKeyValue } = require('../components/utils');
44

55
/**
66
* The Tag DB Service
@@ -31,7 +31,7 @@ const service = {
3131

3232
// dissociate tags that are no longer associated
3333
const dissociateTags = current
34-
.filter(({ key, value}) => !getTagsByKeyValue(tags, key, value));
34+
.filter(({ key, value}) => !getObjectsByKeyValue(tags, key, value));
3535
if (dissociateTags.length) await service.dissociateTags(versionId, dissociateTags, trx);
3636

3737
// associate tags
@@ -71,6 +71,9 @@ const service = {
7171
const associatedTags = await VersionTag.query(trx)
7272
.modify('filterVersionId', versionId);
7373

74+
// TODO: exclude tags (with matching key vand value) that are already joined
75+
// lets us use associateTags in addTags controller
76+
7477
// associate remaining tags
7578
const newJoins = dbTags.filter(({ id }) => {
7679
return !associatedTags.some(({ tagId }) => tagId === id);
@@ -189,8 +192,8 @@ const service = {
189192

190193
tags.forEach(({ key, value }) => {
191194
// if tag is already in db
192-
if (getTagsByKeyValue(allTags, key, value)){
193-
existingTags.push({ id: getTagsByKeyValue(allTags, key, value).id, key: key, value: value });
195+
if (getObjectsByKeyValue(allTags, key, value)){
196+
existingTags.push({ id: getObjectsByKeyValue(allTags, key, value).id, key: key, value: value });
194197
}
195198
// else add to array for inserting
196199
else {

app/src/services/version.js

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,9 @@ const service = {
111111
// https://vincit.github.io/objection.js/recipes/returning-tricks.html
112112
.returning('*')
113113
.throwIfNotFound();
114+
115+
// TODO: prune metadata and tags
116+
114117
if (!etrx) await trx.commit();
115118
return Promise.resolve(response);
116119
} catch (err) {

0 commit comments

Comments
 (0)