Skip to content
This repository was archived by the owner on Mar 13, 2025. It is now read-only.

Commit c405782

Browse files
xxcxyxxcxy
committed
fix contants setting
1 parent bbe24d6 commit c405782

File tree

5 files changed

+170
-76
lines changed

5 files changed

+170
-76
lines changed

‎config/default.js

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,26 @@ module.exports = {
8484
},
8585
organization: {
8686
enrichPolicyName: process.env.ORGANIZATION_ENRICH_POLICYNAME || 'organization-policy'
87+
},
88+
// sub resources under user
89+
achievement: {
90+
userField: process.env.USER_ACHIEVEMENT_PROPERTY_NAME || 'achievements'
91+
},
92+
externalprofile: {
93+
userField: process.env.USER_EXTERNALPROFILE_PROPERTY_NAME || 'externalProfiles'
94+
},
95+
userattribute: {
96+
userField: process.env.USER_ATTRIBUTE_PROPERTY_NAME || 'attributes'
97+
},
98+
userrole: {
99+
userField: process.env.USER_ROLE_PROPERTY_NAME || 'roles'
100+
},
101+
userskill: {
102+
userField: process.env.USER_SKILL_PROPERTY_NAME || 'skills'
103+
},
104+
// sub resources under organization
105+
organizationskillprovider: {
106+
orgField: process.env.ORGANIZATION_SKILLPROVIDER_PROPERTY_NAME || 'skillProviders'
87107
}
88108
}
89109
}

‎src/common/constants.js

Lines changed: 60 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -11,25 +11,21 @@ const topResources = {
1111
type: config.get('ES.ACHIEVEMENT_PROVIDER_TYPE'),
1212
enrich: {
1313
policyName: config.get('ES.ENRICHMENT.achievementprovider.enrichPolicyName'),
14-
enrichFields: ['id', 'achievementsProviderId', 'name', 'uri', 'certifierId', 'certifiedDate', 'created', 'updated', 'createdBy', 'updatedBy']
14+
matchField: 'id',
15+
enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy']
1516
}
1617
},
1718
attribute: {
1819
index: config.get('ES.ATTRIBUTE_INDEX'),
1920
type: config.get('ES.ATTRIBUTE_TYPE'),
2021
enrich: {
2122
policyName: config.get('ES.ENRICHMENT.attribute.enrichPolicyName'),
22-
enrichFields: ['id', 'name', 'attributeGroupId', 'created', 'updated', 'createdBy', 'updatedBy']
23+
matchField: 'id',
24+
enrichFields: ['id', 'name', 'attributeGroupId', 'created', 'updated', 'createdBy', 'updatedBy', 'attributegroup']
2325
},
2426
ingest: {
2527
pipeline: {
26-
id: config.get('ES.ENRICHMENT.attributegroup.pipelineId'),
27-
processors: [{
28-
policyName: config.get('ES.ENRICHMENT.attributegroup.enrichPolicyName'),
29-
field: 'attributegroupId',
30-
targetField: 'attributegroup',
31-
isArray: false
32-
}]
28+
id: config.get('ES.ENRICHMENT.attributegroup.pipelineId')
3329
}
3430
}
3531
},
@@ -38,7 +34,14 @@ const topResources = {
3834
type: config.get('ES.ATTRIBUTE_GROUP_TYPE'),
3935
enrich: {
4036
policyName: config.get('ES.ENRICHMENT.attributegroup.enrichPolicyName'),
41-
enrichFields: ['id', 'organizationId', 'name']
37+
matchField: 'id',
38+
enrichFields: ['id', 'name', 'organizationId', 'created', 'updated', 'createdBy', 'updatedBy']
39+
},
40+
pipeline: {
41+
id: config.get('ES.ENRICHMENT.attributegroup.pipelineId'),
42+
field: 'attributeGroupId',
43+
targetField: 'attributegroup',
44+
maxMatches: '1'
4245
}
4346
},
4447
organization: {
@@ -50,6 +53,7 @@ const topResources = {
5053
type: config.get('ES.ROLE_TYPE'),
5154
enrich: {
5255
policyName: config.get('ES.ENRICHMENT.role.enrichPolicyName'),
56+
matchField: 'id',
5357
enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy']
5458
}
5559
},
@@ -58,17 +62,12 @@ const topResources = {
5862
type: config.get('ES.SKILL_TYPE'),
5963
enrich: {
6064
policyName: config.get('ES.ENRICHMENT.skill.enrichPolicyName'),
61-
enrichFields: ['id', 'skillProviderId', 'name', 'externalId', 'uri', 'created', 'updated', 'createdBy', 'updatedBy']
65+
matchField: 'id',
66+
enrichFields: ['id', 'skillProviderId', 'name', 'externalId', 'uri', 'created', 'updated', 'createdBy', 'updatedBy', 'skillprovider']
6267
},
6368
ingest: {
6469
pipeline: {
65-
id: config.get('ES.ENRICHMENT.skillprovider.pipelineId'),
66-
processors: [{
67-
policyName: config.get('ES.ENRICHMENT.skillprovider.enrichPolicyName'),
68-
field: 'skillProviderId',
69-
targetField: 'skillprovider',
70-
isArray: false
71-
}]
70+
id: config.get('ES.ENRICHMENT.skillprovider.pipelineId')
7271
}
7372
}
7473
},
@@ -77,40 +76,56 @@ const topResources = {
7776
type: config.get('ES.SKILL_PROVIDER_TYPE'),
7877
enrich: {
7978
policyName: config.get('ES.ENRICHMENT.skillprovider.enrichPolicyName'),
80-
enrichFields: ['id', 'name']
79+
matchField: 'id',
80+
enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy']
81+
},
82+
pipeline: {
83+
id: config.get('ES.ENRICHMENT.skillprovider.pipelineId'),
84+
field: 'skillProviderId',
85+
targetField: 'skillprovider',
86+
maxMatches: '1'
8187
}
8288
},
8389
user: {
8490
index: config.get('ES.USER_INDEX'),
8591
type: config.get('ES.USER_TYPE'),
8692
ingest: {
8793
pipeline: {
88-
id: config.get('ES.ENRICHMENT.user.pipelineId'),
89-
processors: [{
90-
policyName: config.get('ES.ENRICHMENT.achievementprovider.enrichPolicyName'),
91-
field: 'achievementsProviderId',
92-
targetField: config.get('ES.USER_ACHIEVEMENT_PROPERTY_NAME'),
93-
isArray: true
94+
id: config.get('ES.ENRICHMENT.user.pipelineId')
95+
}
96+
},
97+
pipeline: {
98+
id: config.get('ES.ENRICHMENT.user.pipelineId'),
99+
processors: [
100+
{
101+
referenceField: config.get('ES.ENRICHMENT.achievement.userField'),
102+
enrichPolicyName: config.get('ES.ENRICHMENT.achievementprovider.enrichPolicyName'),
103+
field: '_ingest._value.achievementsProviderId',
104+
targetField: '_ingest._value.achievementprovider',
105+
maxMatches: '1'
94106
},
95107
{
96-
policyName: config.get('ES.ENRICHMENT.attribute.enrichPolicyName'),
97-
field: 'attributeId',
98-
targetField: config.get('ES.USER_ATTRIBUTE_PROPERTY_NAME'),
99-
isArray: true
108+
referenceField: config.get('ES.ENRICHMENT.userattribute.userField'),
109+
enrichPolicyName: config.get('ES.ENRICHMENT.attribute.enrichPolicyName'),
110+
field: '_ingest._value.attributeId',
111+
targetField: '_ingest._value.attribute',
112+
maxMatches: '1'
100113
},
101114
{
102-
policyName: config.get('ES.ENRICHMENT.role.enrichPolicyName'),
103-
field: 'roleId',
104-
targetField: config.get('ES.USER_ROLE_PROPERTY_NAME'),
105-
isArray: true
115+
referenceField: config.get('ES.ENRICHMENT.userrole.userField'),
116+
enrichPolicyName: config.get('ES.ENRICHMENT.role.enrichPolicyName'),
117+
field: '_ingest._value.roleId',
118+
targetField: '_ingest._value.role',
119+
maxMatches: '1'
106120
},
107121
{
108-
policyName: config.get('ES.ENRICHMENT.skill.enrichPolicyName'),
109-
field: 'skillId',
110-
targetField: config.get('ES.USER_SKILL_PROPERTY_NAME'),
111-
isArray: true
112-
}]
113-
}
122+
referenceField: config.get('ES.ENRICHMENT.userskill.userField'),
123+
enrichPolicyName: config.get('ES.ENRICHMENT.skill.enrichPolicyName'),
124+
field: '_ingest._value.skillId',
125+
targetField: '_ingest._value.skill',
126+
maxMatches: '1'
127+
}
128+
]
114129
}
115130
}
116131
}
@@ -148,7 +163,12 @@ const organizationResources = {
148163
organizationskillprovider: {
149164
propertyName: config.get('ES.ORGANIZATION_SKILLPROVIDER_PROPERTY_NAME'),
150165
relateKey: 'skillProviderId',
151-
validate: payload => validProperties(payload, ['organizationId', 'skillProviderId'])
166+
validate: payload => validProperties(payload, ['organizationId', 'skillProviderId']),
167+
enrich: {
168+
policyName: config.get('ES.ENRICHMENT.organization.enrichPolicyName'),
169+
matchField: 'id',
170+
enrichFields: ['id', 'name', 'created', 'updated', 'createdBy', 'updatedBy', 'skillProviders']
171+
}
152172
}
153173
}
154174

‎test/common/init-es.js

Lines changed: 69 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ const { topResources, userResources } = require('../../src/common/constants')
2020
let client
2121

2222
let needsNestedTypes = ['user']
23+
const enrichedFields = ['attributegroup', 'skillprovider']
2324

2425
/**
2526
* Initialize elastic search index
@@ -44,7 +45,7 @@ const init = async (isForce) => {
4445
index: top.index,
4546
body: {
4647
mappings: {
47-
properties: _(_.get(top, 'enrich.enrichFields', [])).map(p => [p, { type: 'keyword' }]).fromPairs()
48+
properties: _(_.get(top, 'enrich.enrichFields', [])).map(p => [p, { type: _.includes(enrichedFields, p) ? 'nested' : 'keyword' }]).fromPairs()
4849
}
4950
}
5051
})
@@ -74,44 +75,44 @@ const init = async (isForce) => {
7475
body: {
7576
match: {
7677
indices: top.index,
77-
match_field: 'id',
78+
match_field: top.enrich.matchField,
7879
enrich_fields: top.enrich.enrichFields
7980
}
8081
}
8182
})
8283
await client.enrich.executePolicy({ name: top.enrich.policyName })
8384
}
84-
if (top['ingest']) {
85-
_.each(top.ingest.pipeline.processors, processor => {
86-
if (!processors[top.ingest.pipeline.id]) {
87-
processors[top.ingest.pipeline.id] = []
88-
}
89-
if (processor.isArray) {
90-
processors[top.ingest.pipeline.id].push({
85+
if (top.pipeline) {
86+
if (top.pipeline.processors) {
87+
processors[top.pipeline.id] = []
88+
_.each(top.pipeline.processors, processor => {
89+
processors[top.pipeline.id].push({
9190
foreach: {
92-
field: processor.targetField,
91+
field: processor.referenceField,
9392
ignore_missing: true,
9493
processor: {
9594
enrich: {
96-
policy_name: processor.policyName,
95+
policy_name: processor.enrichPolicyName,
9796
ignore_missing: true,
98-
field: `_ingest._value.${processor.field}`,
99-
target_field: '_ingest._value'
97+
field: processor.field,
98+
target_field: processor.targetField,
99+
max_matches: processor.maxMatches
100100
}
101101
}
102102
}
103103
})
104-
} else {
105-
processors[top.ingest.pipeline.id].push({
106-
enrich: {
107-
policy_name: processor.policyName,
108-
ignore_missing: true,
109-
field: processor.field,
110-
target_field: processor.targetField
111-
}
112-
})
113-
}
114-
})
104+
})
105+
} else {
106+
processors[top.pipeline.id] = [{
107+
enrich: {
108+
policy_name: top.enrich.policyName,
109+
ignore_missing: true,
110+
field: top.pipeline.field,
111+
target_field: top.pipeline.targetField,
112+
max_matches: top.pipeline.maxMatches
113+
}
114+
}]
115+
}
115116
}
116117
}
117118

@@ -131,10 +132,10 @@ const init = async (isForce) => {
131132
*/
132133
const clearES = async () => {
133134
for (const key in topResources) {
134-
if (topResources[key].ingest) {
135+
if (topResources[key].pipeline) {
135136
try {
136-
logger.info(`Delete pipeline ${topResources[key].ingest.pipeline.id} if any.`)
137-
await client.ingest.deletePipeline({ id: topResources[key].ingest.pipeline.id })
137+
logger.info(`Delete pipeline ${topResources[key].pipeline.id} if any.`)
138+
await client.ingest.deletePipeline({ id: topResources[key].pipeline.id })
138139
} catch (err) {
139140
// ignore
140141
}
@@ -163,6 +164,44 @@ const clearES = async () => {
163164
}
164165
}
165166

167+
/**
168+
* Check if elastic search is empty
169+
*/
170+
const checkEmpty = async () => {
171+
if (!client) {
172+
client = await helper.getESClient()
173+
}
174+
for (const key in topResources) {
175+
try {
176+
const { body } = await client.search({ index: topResources[key].index })
177+
if (body.hits.total.value > 0) {
178+
return false
179+
}
180+
} catch (err) {
181+
// ignore
182+
}
183+
}
184+
return true
185+
}
186+
187+
/**
188+
* Clear elastic search data
189+
*/
190+
const clearData = async () => {
191+
if (!client) {
192+
client = await helper.getESClient()
193+
}
194+
for (const key in topResources) {
195+
logger.info(`Clear index ${topResources[key].index} data if any.`)
196+
try {
197+
await client.deleteByQuery({ index: topResources[key].index, body: { query: { match_all: {} } } })
198+
} catch (err) {
199+
// ignore
200+
logger.logFullError(err)
201+
}
202+
}
203+
}
204+
166205
if (!module.parent) {
167206
const isForce = process.argv.length === 3 && process.argv[2] === 'force'
168207

@@ -177,5 +216,7 @@ if (!module.parent) {
177216

178217
module.exports = {
179218
init,
180-
clearES
219+
clearES,
220+
checkEmpty,
221+
clearData
181222
}

‎test/common/testHelper.js

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -69,15 +69,25 @@ async function getESGroupRecord (userId, groupId) {
6969
function getExpectValue (payload, relationRecord) {
7070
const result = _.omit(payload, ['resource', 'originalTopic'])
7171
if (topResources[payload.resource] && topResources[payload.resource].ingest) {
72-
_.each(topResources[payload.resource].ingest.pipeline.processors, p => {
73-
const relationResource = _.keys(_.pickBy(topResources, value => value.enrich && value.enrich.policyName === p.policyName))[0]
72+
if (topResources[payload.resource].pipeline) {
73+
_.each(topResources[payload.resource].pipeline.processors, p => {
74+
const relationResource = _.keys(_.pickBy(topResources, value => value.enrich && value.enrich.policyName === p.policyName))[0]
75+
if (relationResource) {
76+
const record = _.find(relationRecord, r => r.payload.resource === relationResource && payload[p.field] === r.payload.id)
77+
if (record) {
78+
result[p.referenceField] = _.pick(record.payload, topResources[relationResource].enrich.enrichFields)
79+
}
80+
}
81+
})
82+
} else {
83+
const relationResource = _.keys(_.pickBy(topResources, value => value.enrich && value.pipeline && value.pipeline.id === topResources[payload.resource].ingest.pipeline.id))[0]
7484
if (relationResource) {
75-
const record = _.find(relationRecord, r => r.payload.resource === relationResource && payload[p.field] === r.payload.id)
85+
const record = _.find(relationRecord, r => r.payload.resource === relationResource && payload[topResources[relationResource].pipeline.field] === r.payload[topResources[relationResource].enrich.matchField])
7686
if (record) {
77-
result[p.targetField] = _.pick(record.payload, topResources[relationResource].enrich.enrichFields)
87+
result[topResources[relationResource].pipeline.targetField] = _.pick(record.payload, topResources[relationResource].enrich.enrichFields)
7888
}
7989
}
80-
})
90+
}
8191
}
8292
return result
8393
}

0 commit comments

Comments
 (0)