Skip to content
This repository was archived by the owner on Mar 13, 2025. It is now read-only.

Commit 985daf9

Browse files
callmekatootiecallmekatootie
authored
Merge pull request #22 from topcoder-platform/Issue_21
Correct the tests
2 parents 55c8582 + 536ea4a commit 985daf9

File tree

10 files changed

+203
-185
lines changed

10 files changed

+203
-185
lines changed

‎README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ Configuration for the tests is at `config/test.js`, only add such new configurat
8989
docker-compose up -d
9090
```
9191

92-
3. initialize Elasticsearch, create configured Elasticsearch index: `npm run init-es force`
92+
3. initialize Elasticsearch. Execute the `insert-data` script in the [API repository](https://github.com/topcoder-platform/u-bahn-api) to set it up and then clear only the data
9393

9494
## Local deployment
9595

‎config/default.js

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,26 @@ module.exports = {
8484
},
8585
organization: {
8686
enrichPolicyName: process.env.ORGANIZATION_ENRICH_POLICYNAME || 'organization-policy'
87+
},
88+
// sub resources under user
89+
achievement: {
90+
userField: process.env.USER_ACHIEVEMENT_PROPERTY_NAME || 'achievements'
91+
},
92+
externalprofile: {
93+
userField: process.env.USER_EXTERNALPROFILE_PROPERTY_NAME || 'externalProfiles'
94+
},
95+
userattribute: {
96+
userField: process.env.USER_ATTRIBUTE_PROPERTY_NAME || 'attributes'
97+
},
98+
userrole: {
99+
userField: process.env.USER_ROLE_PROPERTY_NAME || 'roles'
100+
},
101+
userskill: {
102+
userField: process.env.USER_SKILL_PROPERTY_NAME || 'skills'
103+
},
104+
// sub resources under organization
105+
organizationskillprovider: {
106+
orgField: process.env.ORGANIZATION_SKILLPROVIDER_PROPERTY_NAME || 'skillProviders'
87107
}
88108
}
89109
}

‎package.json

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77
"start": "node src/app.js",
88
"lint": "standard",
99
"lint:fix": "standard --fix",
10-
"init-es": "node test/common/init-es.js",
1110
"view-data": "node test/common/view-data.js",
1211
"test": "mocha test/unit/test.js --require test/unit/prepare.js --timeout 20000 --exit",
1312
"test:cov": "nyc --reporter=html --reporter=text mocha test/unit/test.js --require test/unit/prepare.js --timeout 20000 --exit",

‎src/common/constants.js

Lines changed: 69 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,14 +10,18 @@ const topResources = {
1010
index: config.get('ES.ACHIEVEMENT_PROVIDER_INDEX'),
1111
type: config.get('ES.ACHIEVEMENT_PROVIDER_TYPE'),
1212
enrich: {
13-
policyName: config.get('ES.ENRICHMENT.achievementprovider.enrichPolicyName')
13+
policyName: config.get('ES.ENRICHMENT.achievementprovider.enrichPolicyName'),
14+
matchField: 'id',
15+
enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy']
1416
}
1517
},
1618
attribute: {
1719
index: config.get('ES.ATTRIBUTE_INDEX'),
1820
type: config.get('ES.ATTRIBUTE_TYPE'),
1921
enrich: {
20-
policyName: config.get('ES.ENRICHMENT.attribute.enrichPolicyName')
22+
policyName: config.get('ES.ENRICHMENT.attribute.enrichPolicyName'),
23+
matchField: 'id',
24+
enrichFields: ['id', 'name', 'attributeGroupId', 'created', 'updated', 'createdBy', 'updatedBy', 'attributegroup']
2125
},
2226
ingest: {
2327
pipeline: {
@@ -29,7 +33,15 @@ const topResources = {
2933
index: config.get('ES.ATTRIBUTE_GROUP_INDEX'),
3034
type: config.get('ES.ATTRIBUTE_GROUP_TYPE'),
3135
enrich: {
32-
policyName: config.get('ES.ENRICHMENT.attributegroup.enrichPolicyName')
36+
policyName: config.get('ES.ENRICHMENT.attributegroup.enrichPolicyName'),
37+
matchField: 'id',
38+
enrichFields: ['id', 'name', 'organizationId', 'created', 'updated', 'createdBy', 'updatedBy']
39+
},
40+
pipeline: {
41+
id: config.get('ES.ENRICHMENT.attributegroup.pipelineId'),
42+
field: 'attributeGroupId',
43+
targetField: 'attributegroup',
44+
maxMatches: '1'
3345
}
3446
},
3547
organization: {
@@ -40,14 +52,18 @@ const topResources = {
4052
index: config.get('ES.ROLE_INDEX'),
4153
type: config.get('ES.ROLE_TYPE'),
4254
enrich: {
43-
policyName: config.get('ES.ENRICHMENT.role.enrichPolicyName')
55+
policyName: config.get('ES.ENRICHMENT.role.enrichPolicyName'),
56+
matchField: 'id',
57+
enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy']
4458
}
4559
},
4660
skill: {
4761
index: config.get('ES.SKILL_INDEX'),
4862
type: config.get('ES.SKILL_TYPE'),
4963
enrich: {
50-
policyName: config.get('ES.ENRICHMENT.skill.enrichPolicyName')
64+
policyName: config.get('ES.ENRICHMENT.skill.enrichPolicyName'),
65+
matchField: 'id',
66+
enrichFields: ['id', 'skillProviderId', 'name', 'externalId', 'uri', 'created', 'updated', 'createdBy', 'updatedBy', 'skillprovider']
5167
},
5268
ingest: {
5369
pipeline: {
@@ -59,7 +75,15 @@ const topResources = {
5975
index: config.get('ES.SKILL_PROVIDER_INDEX'),
6076
type: config.get('ES.SKILL_PROVIDER_TYPE'),
6177
enrich: {
62-
policyName: config.get('ES.ENRICHMENT.skillprovider.enrichPolicyName')
78+
policyName: config.get('ES.ENRICHMENT.skillprovider.enrichPolicyName'),
79+
matchField: 'id',
80+
enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy']
81+
},
82+
pipeline: {
83+
id: config.get('ES.ENRICHMENT.skillprovider.pipelineId'),
84+
field: 'skillProviderId',
85+
targetField: 'skillprovider',
86+
maxMatches: '1'
6387
}
6488
},
6589
user: {
@@ -69,6 +93,39 @@ const topResources = {
6993
pipeline: {
7094
id: config.get('ES.ENRICHMENT.user.pipelineId')
7195
}
96+
},
97+
pipeline: {
98+
id: config.get('ES.ENRICHMENT.user.pipelineId'),
99+
processors: [
100+
{
101+
referenceField: config.get('ES.ENRICHMENT.achievement.userField'),
102+
enrichPolicyName: config.get('ES.ENRICHMENT.achievementprovider.enrichPolicyName'),
103+
field: '_ingest._value.achievementsProviderId',
104+
targetField: '_ingest._value.achievementprovider',
105+
maxMatches: '1'
106+
},
107+
{
108+
referenceField: config.get('ES.ENRICHMENT.userattribute.userField'),
109+
enrichPolicyName: config.get('ES.ENRICHMENT.attribute.enrichPolicyName'),
110+
field: '_ingest._value.attributeId',
111+
targetField: '_ingest._value.attribute',
112+
maxMatches: '1'
113+
},
114+
{
115+
referenceField: config.get('ES.ENRICHMENT.userrole.userField'),
116+
enrichPolicyName: config.get('ES.ENRICHMENT.role.enrichPolicyName'),
117+
field: '_ingest._value.roleId',
118+
targetField: '_ingest._value.role',
119+
maxMatches: '1'
120+
},
121+
{
122+
referenceField: config.get('ES.ENRICHMENT.userskill.userField'),
123+
enrichPolicyName: config.get('ES.ENRICHMENT.skill.enrichPolicyName'),
124+
field: '_ingest._value.skillId',
125+
targetField: '_ingest._value.skill',
126+
maxMatches: '1'
127+
}
128+
]
72129
}
73130
}
74131
}
@@ -106,7 +163,12 @@ const organizationResources = {
106163
organizationskillprovider: {
107164
propertyName: config.get('ES.ORGANIZATION_SKILLPROVIDER_PROPERTY_NAME'),
108165
relateKey: 'skillProviderId',
109-
validate: payload => validProperties(payload, ['organizationId', 'skillProviderId'])
166+
validate: payload => validProperties(payload, ['organizationId', 'skillProviderId']),
167+
enrich: {
168+
policyName: config.get('ES.ENRICHMENT.organization.enrichPolicyName'),
169+
matchField: 'id',
170+
enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy', 'skillProviders']
171+
}
110172
}
111173
}
112174

‎test/common/init-es.js

Lines changed: 19 additions & 114 deletions
Original file line numberDiff line numberDiff line change
@@ -11,146 +11,51 @@
1111
* node src/init-es
1212
* node src/init-es force
1313
*/
14-
15-
const config = require('config')
16-
const _ = require('lodash')
1714
const logger = require('../../src/common/logger')
1815
const helper = require('../../src/common/helper')
19-
const { topResources, userResources } = require('../../src/common/constants')
16+
const { topResources } = require('../../src/common/constants')
2017

2118
let client
2219

23-
let needsNestedTypes = ['user']
24-
2520
/**
26-
* Initialize elastic search index
27-
* @param {Boolean} isForce boolean flag indicate it is forced operation
21+
* Check if elastic search is empty
2822
*/
29-
const init = async (isForce) => {
23+
const checkEmpty = async () => {
3024
if (!client) {
3125
client = await helper.getESClient()
3226
}
33-
if (isForce) {
34-
await clearES()
35-
}
3627
for (const key in topResources) {
37-
const exists = await client.indices.exists({ index: topResources[key].index })
38-
if (exists.body) {
39-
logger.info(`The index ${topResources[key].index} exists.`)
40-
} else {
41-
logger.info(`The index ${topResources[key].index} will be created.`)
42-
await client.indices.create({
43-
index: topResources[key].index,
44-
body: {
45-
mappings: {
46-
properties: _(topResources[key].mappingFields).map(p => [p, { type: 'keyword' }]).fromPairs()
47-
}
48-
}
49-
})
50-
if (needsNestedTypes.includes(key)) {
51-
for (const childKey in userResources) {
52-
if (userResources[childKey].isNested) {
53-
await client.indices.putMapping({
54-
index: topResources[key].index,
55-
type: topResources[key].type,
56-
include_type_name: true,
57-
body: {
58-
properties: {
59-
[userResources[childKey].propertyName]: {
60-
type: 'nested'
61-
}
62-
}
63-
}
64-
})
65-
}
66-
}
28+
try {
29+
const { body } = await client.search({ index: topResources[key].index })
30+
if (body.hits.total.value > 0) {
31+
return false
6732
}
33+
} catch (err) {
34+
// ignore
6835
}
6936
}
70-
const processors = []
71-
for (const key in userResources) {
72-
logger.info(`The enrich policy ${key}-policy will be created.`)
73-
const top = topResources[userResources[key].relateTopResource]
74-
await client.enrich.putPolicy({
75-
name: top.enrichPolicy,
76-
body: {
77-
match: {
78-
indices: top.index,
79-
match_field: 'id',
80-
enrich_fields: top.mappingFields
81-
}
82-
}
83-
})
84-
await client.enrich.executePolicy({ name: top.enrichPolicy })
85-
processors.push({
86-
foreach: {
87-
field: userResources[key].propertyName,
88-
ignore_missing: true,
89-
processor: {
90-
enrich: {
91-
policy_name: top.enrichPolicy,
92-
ignore_missing: true,
93-
field: `_ingest._value.${userResources[key].relateKey}`,
94-
target_field: '_ingest._value'
95-
}
96-
}
97-
}
98-
})
99-
}
100-
101-
logger.info(`The pipeline ${config.ES.ENRICH_USER_PIPELINE_NAME} will be created.`)
102-
await client.ingest.putPipeline({
103-
id: config.ES.ENRICH_USER_PIPELINE_NAME,
104-
body: {
105-
processors
106-
}
107-
})
37+
return true
10838
}
10939

11040
/**
111-
* Delete elastic search index
41+
* Clear elastic search data
11242
*/
113-
const clearES = async () => {
114-
try {
115-
logger.info(`Delete pipeline ${config.ES.ENRICH_USER_PIPELINE_NAME} if any.`)
116-
await client.ingest.deletePipeline({ id: config.ES.ENRICH_USER_PIPELINE_NAME })
117-
} catch (err) {
118-
// ignore
119-
}
120-
for (const key in userResources) {
121-
try {
122-
const policyName = topResources[userResources[key].relateTopResource].enrichPolicy
123-
logger.info(`Delete enrich policy ${policyName} if any.`)
124-
await client.enrich.deletePolicy({
125-
name: policyName
126-
})
127-
} catch (err) {
128-
// ignore
129-
}
43+
const clearData = async () => {
44+
if (!client) {
45+
client = await helper.getESClient()
13046
}
13147
for (const key in topResources) {
132-
logger.info(`Delete index ${topResources[key].index} if any.`)
48+
logger.info(`Clear index ${topResources[key].index} data if any.`)
13349
try {
134-
await client.indices.delete({ index: topResources[key].index })
50+
await client.deleteByQuery({ index: topResources[key].index, body: { query: { match_all: {} } } })
13551
} catch (err) {
13652
// ignore
53+
logger.logFullError(err)
13754
}
13855
}
13956
}
14057

141-
if (!module.parent) {
142-
const isForce = process.argv.length === 3 && process.argv[2] === 'force'
143-
144-
init(isForce).then(() => {
145-
logger.info('done')
146-
process.exit()
147-
}).catch((e) => {
148-
logger.error(e)
149-
process.exit()
150-
})
151-
}
152-
15358
module.exports = {
154-
init,
155-
clearES
59+
checkEmpty,
60+
clearData
15661
}

0 commit comments

Comments
 (0)