Skip to content

Commit 894b6aa

Browse files
bshafferbshaffer
authored
fix: dlp samples for grpc (GoogleCloudPlatform#1123)
1 parent 98dc0e7 commit 894b6aa

15 files changed

+296
-233
lines changed

‎.kokoro/secrets-example.sh‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,18 @@
11
#!/bin/bash
22

3-
# This file contains the necessary environment variables for the kokoro
3+
# This file contains the necessary environment variables for the kokoro
44
# tests. Contact the repository owners if you need access to view or modify
55
# the variables.
6-
#
6+
#
77
# Run the following gcloud command to decrypt secrets.sh.enc as follows:
88
#
99
# gcloud kms decrypt --location=global --keyring=ci --key=ci \
1010
# --ciphertext-file=.kokoro/secrets.sh.enc \
1111
# --plaintext-file=.kokoro/secrets.sh
1212
#
1313
# Then run `source .kokoro/secrets.sh`
14-
#
15-
# To modify the file, edit .kokoro/secrets.sh then use the following gcloud
14+
#
15+
# To modify the file, edit .kokoro/secrets.sh then use the following gcloud
1616
# command to encrypt it with the changes:
1717
#
1818
# gcloud kms encrypt --location=global --keyring=ci --key=ci \
@@ -66,8 +66,6 @@ export CLOUD_DATASTORE_NAMESPACE=
6666
export DATASTORE_EVENTUALLY_CONSISTENT_RETRY_COUNT=
6767

6868
# DLP
69-
export DLP_TOPIC=dlp-tests
70-
export DLP_SUBSCRIPTION=dlp-tests
7169
export DLP_DEID_WRAPPED_KEY=
7270
export DLP_DEID_KEY_NAME=projects/$GOOGLE_PROJECT_ID/locations/global/keyRings/ci/cryptoKeys/ci
7371

‎.kokoro/secrets.sh.enc‎

-61 Bytes
Binary file not shown.

‎dlp/src/categorical_stats.php‎

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,6 @@
3434
/**
3535
* Computes risk metrics of a column of data in a Google BigQuery table.
3636
*/
37-
use Google\Cloud\Core\ExponentialBackoff;
3837
use Google\Cloud\Dlp\V2\DlpServiceClient;
3938
use Google\Cloud\Dlp\V2\RiskAnalysisJobConfig;
4039
use Google\Cloud\Dlp\V2\BigQueryTable;
@@ -94,7 +93,7 @@
9493
->setActions([$action]);
9594

9695
// Submit request
97-
$parent = "projects/$callingProjectId/locations/global";;
96+
$parent = "projects/$callingProjectId/locations/global";
9897
$job = $dlp->createDlpJob($parent, [
9998
'riskJob' => $riskJob
10099
]);
@@ -103,9 +102,10 @@
103102
$subscription = $topic->subscription($subscriptionId);
104103

105104
// Poll Pub/Sub using exponential backoff until job finishes
106-
$backoff = new ExponentialBackoff(30);
107-
$backoff->execute(function () use ($subscription, $dlp, &$job) {
108-
printf('Waiting for job to complete' . PHP_EOL);
105+
// Consider using an asynchronous execution model such as Cloud Functions
106+
$attempt = 1;
107+
$startTime = time();
108+
do {
109109
foreach ($subscription->pull() as $message) {
110110
if (isset($message->attributes()['DlpJobName']) &&
111111
$message->attributes()['DlpJobName'] === $job->getName()) {
@@ -114,11 +114,13 @@
114114
do {
115115
$job = $dlp->getDlpJob($job->getName());
116116
} while ($job->getState() == JobState::RUNNING);
117-
return true;
117+
break 2; // break from parent do while
118118
}
119119
}
120-
throw new Exception('Job has not yet completed');
121-
});
120+
printf('Waiting for job to complete' . PHP_EOL);
121+
// Exponential backoff with max delay of 60 seconds
122+
sleep(min(60, pow(2, ++$attempt)));
123+
} while (time() - $startTime < 600); // 10 minute timeout
122124

123125
// Print finding counts
124126
printf('Job %s status: %s' . PHP_EOL, $job->getName(), JobState::name($job->getState()));
@@ -151,6 +153,9 @@
151153
var_dump($error->getDetails());
152154
}
153155
break;
156+
case JobState::PENDING:
157+
printf('Job has not completed. Consider a longer timeout or an asynchronous execution model' . PHP_EOL);
158+
break;
154159
default:
155160
printf('Unexpected job state.');
156161
}

‎dlp/src/delete_inspect_template.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@
4444
$dlp = new DlpServiceClient();
4545

4646
// Run template deletion request
47-
$templateName = $dlp->projectInspectTemplateName($callingProjectId, $templateId);
47+
$templateName = "projects/$callingProjectId/locations/global/inspectTemplates/$templateId";
4848
$dlp->deleteInspectTemplate($templateName);
4949

5050
// Print results

‎dlp/src/delete_trigger.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@
4545

4646
// Run request
4747
// The Parent project ID is automatically extracted from this parameter
48-
$triggerName = $dlp->projectJobTriggerName($callingProjectId, $triggerId);
48+
$triggerName = "projects/$callingProjectId/locations/global/jobTriggers/$triggerId";
4949
$response = $dlp->deleteJobTrigger($triggerName);
5050

5151
// Print the results

‎dlp/src/inspect_bigquery.php‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@
3535
/**
3636
* Inspect a BigQuery table , using Pub/Sub for job status notifications.
3737
*/
38-
use Google\Cloud\Core\ExponentialBackoff;
3938
use Google\Cloud\Dlp\V2\DlpServiceClient;
4039
use Google\Cloud\Dlp\V2\BigQueryOptions;
4140
use Google\Cloud\Dlp\V2\InfoType;
@@ -120,9 +119,10 @@
120119
]);
121120

122121
// Poll Pub/Sub using exponential backoff until job finishes
123-
$backoff = new ExponentialBackoff(30);
124-
$backoff->execute(function () use ($subscription, $dlp, &$job) {
125-
printf('Waiting for job to complete' . PHP_EOL);
122+
// Consider using an asynchronous execution model such as Cloud Functions
123+
$attempt = 1;
124+
$startTime = time();
125+
do {
126126
foreach ($subscription->pull() as $message) {
127127
if (isset($message->attributes()['DlpJobName']) &&
128128
$message->attributes()['DlpJobName'] === $job->getName()) {
@@ -131,11 +131,13 @@
131131
do {
132132
$job = $dlp->getDlpJob($job->getName());
133133
} while ($job->getState() == JobState::RUNNING);
134-
return true;
134+
break 2; // break from parent do while
135135
}
136136
}
137-
throw new Exception('Job has not yet completed');
138-
});
137+
printf('Waiting for job to complete' . PHP_EOL);
138+
// Exponential backoff with max delay of 60 seconds
139+
sleep(min(60, pow(2, ++$attempt)));
140+
} while (time() - $startTime < 600); // 10 minute timeout
139141

140142
// Print finding counts
141143
printf('Job %s status: %s' . PHP_EOL, $job->getName(), JobState::name($job->getState()));
@@ -161,6 +163,9 @@
161163
var_dump($error->getDetails());
162164
}
163165
break;
166+
case JobState::PENDING:
167+
printf('Job has not completed. Consider a longer timeout or an asynchronous execution model' . PHP_EOL);
168+
break;
164169
default:
165170
printf('Unexpected job state. Most likely, the job is either running or has not yet started.');
166171
}

‎dlp/src/inspect_datastore.php‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@
3535
/**
3636
* Inspect Datastore, using Pub/Sub for job status notifications.
3737
*/
38-
use Google\Cloud\Core\ExponentialBackoff;
3938
use Google\Cloud\Dlp\V2\DlpServiceClient;
4039
use Google\Cloud\Dlp\V2\DatastoreOptions;
4140
use Google\Cloud\Dlp\V2\InfoType;
@@ -124,9 +123,10 @@
124123
]);
125124

126125
// Poll Pub/Sub using exponential backoff until job finishes
127-
$backoff = new ExponentialBackoff(30);
128-
$backoff->execute(function () use ($subscription, $dlp, &$job) {
129-
printf('Waiting for job to complete' . PHP_EOL);
126+
// Consider using an asynchronous execution model such as Cloud Functions
127+
$attempt = 1;
128+
$startTime = time();
129+
do {
130130
foreach ($subscription->pull() as $message) {
131131
if (isset($message->attributes()['DlpJobName']) &&
132132
$message->attributes()['DlpJobName'] === $job->getName()) {
@@ -135,11 +135,13 @@
135135
do {
136136
$job = $dlp->getDlpJob($job->getName());
137137
} while ($job->getState() == JobState::RUNNING);
138-
return true;
138+
break 2; // break from parent do while
139139
}
140140
}
141-
throw new Exception('Job has not yet completed');
142-
});
141+
printf('Waiting for job to complete' . PHP_EOL);
142+
// Exponential backoff with max delay of 60 seconds
143+
sleep(min(60, pow(2, ++$attempt)));
144+
} while (time() - $startTime < 600); // 10 minute timeout
143145

144146
// Print finding counts
145147
printf('Job %s status: %s' . PHP_EOL, $job->getName(), JobState::name($job->getState()));
@@ -161,6 +163,9 @@
161163
var_dump($error->getDetails());
162164
}
163165
break;
166+
case JobState::PENDING:
167+
printf('Job has not completed. Consider a longer timeout or an asynchronous execution model' . PHP_EOL);
168+
break;
164169
default:
165170
print('Unexpected job state.');
166171
}

‎dlp/src/inspect_gcs.php‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@
3535
/**
3636
* Inspect a file stored on Google Cloud Storage , using Pub/Sub for job status notifications.
3737
*/
38-
use Google\Cloud\Core\ExponentialBackoff;
3938
use Google\Cloud\Dlp\V2\DlpServiceClient;
4039
use Google\Cloud\Dlp\V2\CloudStorageOptions;
4140
use Google\Cloud\Dlp\V2\CloudStorageOptions\FileSet;
@@ -120,9 +119,10 @@
120119
]);
121120

122121
// Poll Pub/Sub using exponential backoff until job finishes
123-
$backoff = new ExponentialBackoff(30);
124-
$backoff->execute(function () use ($subscription, $dlp, &$job) {
125-
printf('Waiting for job to complete' . PHP_EOL);
122+
// Consider using an asynchronous execution model such as Cloud Functions
123+
$attempt = 1;
124+
$startTime = time();
125+
do {
126126
foreach ($subscription->pull() as $message) {
127127
if (isset($message->attributes()['DlpJobName']) &&
128128
$message->attributes()['DlpJobName'] === $job->getName()) {
@@ -131,11 +131,13 @@
131131
do {
132132
$job = $dlp->getDlpJob($job->getName());
133133
} while ($job->getState() == JobState::RUNNING);
134-
return true;
134+
break 2; // break from parent do while
135135
}
136136
}
137-
throw new Exception('Job has not yet completed');
138-
});
137+
printf('Waiting for job to complete' . PHP_EOL);
138+
// Exponential backoff with max delay of 60 seconds
139+
sleep(min(60, pow(2, ++$attempt)));
140+
} while (time() - $startTime < 600); // 10 minute timeout
139141

140142
// Print finding counts
141143
printf('Job %s status: %s' . PHP_EOL, $job->getName(), JobState::name($job->getState()));
@@ -157,6 +159,9 @@
157159
var_dump($error->getDetails());
158160
}
159161
break;
162+
case JobState::PENDING:
163+
printf('Job has not completed. Consider a longer timeout or an asynchronous execution model' . PHP_EOL);
164+
break;
160165
default:
161166
print('Unexpected job state. Most likely, the job is either running or has not yet started.');
162167
}

‎dlp/src/k_anonymity.php‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,6 @@
3434
/**
3535
* Computes the k-anonymity of a column set in a Google BigQuery table.
3636
*/
37-
use Google\Cloud\Core\ExponentialBackoff;
3837
use Google\Cloud\Dlp\V2\DlpServiceClient;
3938
use Google\Cloud\Dlp\V2\RiskAnalysisJobConfig;
4039
use Google\Cloud\Dlp\V2\BigQueryTable;
@@ -107,9 +106,10 @@ function ($id) {
107106
]);
108107

109108
// Poll Pub/Sub using exponential backoff until job finishes
110-
$backoff = new ExponentialBackoff(30);
111-
$backoff->execute(function () use ($subscription, $dlp, &$job) {
112-
printf('Waiting for job to complete' . PHP_EOL);
109+
// Consider using an asynchronous execution model such as Cloud Functions
110+
$attempt = 1;
111+
$startTime = time();
112+
do {
113113
foreach ($subscription->pull() as $message) {
114114
if (isset($message->attributes()['DlpJobName']) &&
115115
$message->attributes()['DlpJobName'] === $job->getName()) {
@@ -118,11 +118,13 @@ function ($id) {
118118
do {
119119
$job = $dlp->getDlpJob($job->getName());
120120
} while ($job->getState() == JobState::RUNNING);
121-
return true;
121+
break 2; // break from parent do while
122122
}
123123
}
124-
throw new Exception('Job has not yet completed');
125-
});
124+
printf('Waiting for job to complete' . PHP_EOL);
125+
// Exponential backoff with max delay of 60 seconds
126+
sleep(min(60, pow(2, ++$attempt)));
127+
} while (time() - $startTime < 600); // 10 minute timeout
126128

127129
// Print finding counts
128130
printf('Job %s status: %s' . PHP_EOL, $job->getName(), JobState::name($job->getState()));
@@ -161,6 +163,9 @@ function ($id) {
161163
var_dump($error->getDetails());
162164
}
163165
break;
166+
case JobState::PENDING:
167+
printf('Job has not completed. Consider a longer timeout or an asynchronous execution model' . PHP_EOL);
168+
break;
164169
default:
165170
printf('Unexpected job state. Most likely, the job is either running or has not yet started.');
166171
}

‎dlp/src/k_map.php‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
return print("Usage: php k_map.php CALLING_PROJECT DATA_PROJECT TOPIC SUBSCRIPTION DATASET TABLE REGION_CODE QUASI_ID_NAMES INFO_TYPES\n");
3030
}
3131
list($_, $callingProjectId, $dataProjectId, $topicId, $subscriptionId, $datasetId, $tableId, $regionCode, $quasiIdNames, $infoTypes) = $argv;
32+
3233
// Convert comma-separated lists to arrays
3334
$quasiIdNames = explode(',', $quasiIdNames);
3435
$infoTypes = explode(',', $infoTypes);
@@ -37,7 +38,6 @@
3738
/**
3839
* Computes the k-map risk estimation of a column set in a Google BigQuery table.
3940
*/
40-
use Google\Cloud\Core\ExponentialBackoff;
4141
use Google\Cloud\Dlp\V2\DlpServiceClient;
4242
use Google\Cloud\Dlp\V2\InfoType;
4343
use Google\Cloud\Dlp\V2\RiskAnalysisJobConfig;
@@ -128,9 +128,10 @@
128128
]);
129129

130130
// Poll Pub/Sub using exponential backoff until job finishes
131-
$backoff = new ExponentialBackoff(30);
132-
$backoff->execute(function () use ($subscription, $dlp, &$job) {
133-
printf('Waiting for job to complete' . PHP_EOL);
131+
// Consider using an asynchronous execution model such as Cloud Functions
132+
$attempt = 1;
133+
$startTime = time();
134+
do {
134135
foreach ($subscription->pull() as $message) {
135136
if (isset($message->attributes()['DlpJobName']) &&
136137
$message->attributes()['DlpJobName'] === $job->getName()) {
@@ -139,11 +140,13 @@
139140
do {
140141
$job = $dlp->getDlpJob($job->getName());
141142
} while ($job->getState() == JobState::RUNNING);
142-
return true;
143+
break 2; // break from parent do while
143144
}
144145
}
145-
throw new Exception('Job has not yet completed');
146-
});
146+
printf('Waiting for job to complete' . PHP_EOL);
147+
// Exponential backoff with max delay of 60 seconds
148+
sleep(min(60, pow(2, ++$attempt)));
149+
} while (time() - $startTime < 600); // 10 minute timeout
147150

148151
// Print finding counts
149152
printf('Job %s status: %s' . PHP_EOL, $job->getName(), JobState::name($job->getState()));
@@ -183,6 +186,9 @@
183186
var_dump($error->getDetails());
184187
}
185188
break;
189+
case JobState::PENDING:
190+
printf('Job has not completed. Consider a longer timeout or an asynchronous execution model' . PHP_EOL);
191+
break;
186192
default:
187193
print('Unexpected job state. Most likely, the job is either running or has not yet started.');
188194
}

0 commit comments

Comments
 (0)