Skip to content

Commit 3cadc8e

Browse files
amanda-tarafajskeet
amanda-tarafa
authored and
jskeet
committed
Modifies insert methods to return BigQueryInsertResults.
1 parent 9a2f966 commit 3cadc8e

7 files changed

Lines changed: 242 additions & 124 deletions

File tree

‎apis/Google.Cloud.BigQuery.V2/Google.Cloud.BigQuery.V2.GenerateOverloads/Methods/InsertData.xml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,7 @@
133133
<Method Name="InsertRows"
134134
RegionLabel="InsertRows(sequence)"
135135
TargetType="Table"
136-
ReturnType="void">
136+
ReturnType="BigQueryInsertResults">
137137

138138
<Options Type="InsertOptions" />
139139
<AdditionalParameters>
@@ -144,6 +144,7 @@
144144
<summary>
145145
Inserts all the given rows of data into {target}.
146146
</summary>
147+
<returns>An insert result object which contains information on insert errors if any.</returns>
147148
</Comments>
148149
</Method>
149150
</File>

‎apis/Google.Cloud.BigQuery.V2/Google.Cloud.BigQuery.V2.IntegrationTests/BigQueryFixture.cs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -263,15 +263,17 @@ private void CreateExhaustiveTypesTable(BigQueryDataset dataset)
263263
InsertAndWait(table, () => table.InsertRow(ExhaustiveTypesTest.GetSampleRow()), 1);
264264
}
265265

266-
internal void InsertAndWait(BigQueryTable table, Action insertAction, int expectedRowCountChange)
266+
internal BigQueryInsertResults InsertAndWait(BigQueryTable table, Func<BigQueryInsertResults> insertAction, int expectedRowCountChange)
267267
{
268268
var countBefore = table.ListRows().Count();
269269
var expectedCount = countBefore + expectedRowCountChange;
270-
insertAction();
270+
var results = insertAction();
271271
// Wait until there are *at least* enough rows
272272
int actualCount = table.PollUntilRowCountIsAtLeast(expectedCount);
273273
// Now check it's *exactly* the right number of rows.
274274
Assert.Equal(expectedCount, actualCount);
275+
276+
return results;
275277
}
276278

277279
internal List<string> LoadTextResource(string relativeName)

‎apis/Google.Cloud.BigQuery.V2/Google.Cloud.BigQuery.V2.IntegrationTests/InsertTest.cs‎

Lines changed: 76 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,12 @@ public InsertTest(BigQueryFixture fixture)
3030
_fixture = fixture;
3131
}
3232

33+
private void AssertAllRowsInserted(BigQueryInsertResults insertResult)
34+
{
35+
Assert.Equal(BigQueryInsertStatus.AllRowsInserted, insertResult.Status);
36+
Assert.Empty(insertResult.Errors);
37+
}
38+
3339
[Fact]
3440
public void InsertRow()
3541
{
@@ -39,7 +45,9 @@ public void InsertRow()
3945

4046
var row = BuildRow("Joe", 100, new DateTime(2016, 4, 26, 11, 43, 1, DateTimeKind.Utc));
4147

42-
_fixture.InsertAndWait(table, () => table.InsertRow(row), 1);
48+
var insertResult = _fixture.InsertAndWait(table, () => table.InsertRow(row), 1);
49+
50+
AssertAllRowsInserted(insertResult);
4351

4452
var rowsAfter = table.ListRows();
4553
var fetched = rowsAfter.Single(r => (string)r["player"] == "Joe");
@@ -60,7 +68,9 @@ public void InsertRows()
6068
BuildRow("Lisa", 90, new DateTime(2011, 10, 12, 0, 0, 0, DateTimeKind.Utc))
6169
};
6270

63-
_fixture.InsertAndWait(table, () => table.InsertRows(rows), 2);
71+
var insertResult = _fixture.InsertAndWait(table, () => table.InsertRows(rows), 2);
72+
73+
AssertAllRowsInserted(insertResult);
6474

6575
var rowsAfter = table.ListRows().ToList();
6676
Assert.Contains(rowsAfter, r => (string)r["player"] == "Jenny");
@@ -81,11 +91,13 @@ public void InsertRows_AllowEmptyInsertIds()
8191
BuildRow("Henry", 90, new DateTime(2011, 10, 12, 0, 0, 0, DateTimeKind.Utc))
8292
};
8393

84-
_fixture.InsertAndWait(table, () => table.InsertRows(rows, options), 2);
94+
var insertResult = _fixture.InsertAndWait(table, () => table.InsertRows(rows, options), 2);
8595

8696
Assert.Null(rows[0].InsertId);
8797
Assert.Null(rows[1].InsertId);
8898

99+
AssertAllRowsInserted(insertResult);
100+
89101
var rowsAfter = table.ListRows().ToList();
90102
Assert.Contains(rowsAfter, r => (string)r["player"] == "Helen");
91103
Assert.Contains(rowsAfter, r => (string)r["player"] == "Henry");
@@ -95,22 +107,25 @@ public static IEnumerable<object[]> BadDataThrowsOptions
95107
{
96108
get
97109
{
98-
yield return new object[] { null };
99-
yield return new object[] { new InsertOptions() };
100-
yield return new object[] { new InsertOptions { SkipInvalidRows = false } };
101-
yield return new object[] { new InsertOptions { SkipInvalidRows = true } };
102-
yield return new object[] { new InsertOptions { AllowUnknownFields = false } };
103-
yield return new object[] { new InsertOptions { AllowUnknownFields = true } };
104-
yield return new object[] { new InsertOptions { SkipInvalidRows = false, AllowUnknownFields = false } };
105-
yield return new object[] { new InsertOptions { SkipInvalidRows = false, AllowUnknownFields = true } };
106-
yield return new object[] { new InsertOptions { SkipInvalidRows = true, AllowUnknownFields = false } };
107-
yield return new object[] { new InsertOptions { SkipInvalidRows = true, AllowUnknownFields = true } };
110+
int[] bothRowsIndexes = new int[] { 0, 1 };
111+
int[] oneRowIndex = new int[] { 1 };
112+
113+
yield return new object[] { null, bothRowsIndexes };
114+
yield return new object[] { new InsertOptions(), bothRowsIndexes };
115+
yield return new object[] { new InsertOptions { SkipInvalidRows = false }, bothRowsIndexes };
116+
yield return new object[] { new InsertOptions { SkipInvalidRows = true }, bothRowsIndexes };
117+
yield return new object[] { new InsertOptions { AllowUnknownFields = false }, bothRowsIndexes };
118+
yield return new object[] { new InsertOptions { AllowUnknownFields = true }, bothRowsIndexes };
119+
yield return new object[] { new InsertOptions { SkipInvalidRows = false, AllowUnknownFields = false }, bothRowsIndexes };
120+
yield return new object[] { new InsertOptions { SkipInvalidRows = false, AllowUnknownFields = true }, bothRowsIndexes };
121+
yield return new object[] { new InsertOptions { SkipInvalidRows = true, AllowUnknownFields = false }, bothRowsIndexes };
122+
yield return new object[] { new InsertOptions { SkipInvalidRows = true, AllowUnknownFields = true }, oneRowIndex };
108123
}
109124
}
110125

111126
[Theory]
112127
[MemberData(nameof(BadDataThrowsOptions))]
113-
public void InsertRow_BadData_Throws(InsertOptions options)
128+
public void InsertRow_BadData_Throws(InsertOptions options, int[] errorRowsIndexes)
114129
{
115130
var client = BigQueryClient.Create(_fixture.ProjectId);
116131
var dataset = client.GetDataset(_fixture.DatasetId);
@@ -123,28 +138,37 @@ public void InsertRow_BadData_Throws(InsertOptions options)
123138
new BigQueryInsertRow { { "noSuchField", 10 } },
124139
new BigQueryInsertRow { {"year", "Unknown"} }
125140
};
126-
Assert.Throws<GoogleApiException>(() => table.InsertRows(rows, options));
141+
var exception = Assert.Throws<GoogleApiException>(() => table.InsertRows(rows, options));
142+
143+
Assert.Equal(errorRowsIndexes.Length, exception.Error.Errors.Count);
144+
foreach (var index in errorRowsIndexes)
145+
{
146+
Assert.Contains(exception.Error.Errors, e => e.Message.ToLower().Contains($"in row {index}"));
147+
}
127148
}
128149

129150
public static IEnumerable<object[]> BadDataSilentOptions
130151
{
131152
get
132153
{
133-
yield return new object[] { new InsertOptions { SuppressInsertErrors = true } };
134-
yield return new object[] { new InsertOptions { SkipInvalidRows = false, SuppressInsertErrors = true } };
135-
yield return new object[] { new InsertOptions { SkipInvalidRows = true, SuppressInsertErrors = true } };
136-
yield return new object[] { new InsertOptions { AllowUnknownFields = false, SuppressInsertErrors = true } };
137-
yield return new object[] { new InsertOptions { AllowUnknownFields = true, SuppressInsertErrors = true } };
138-
yield return new object[] { new InsertOptions { SkipInvalidRows = false, AllowUnknownFields = false, SuppressInsertErrors = true } };
139-
yield return new object[] { new InsertOptions { SkipInvalidRows = false, AllowUnknownFields = true, SuppressInsertErrors = true } };
140-
yield return new object[] { new InsertOptions { SkipInvalidRows = true, AllowUnknownFields = false, SuppressInsertErrors = true } };
141-
yield return new object[] { new InsertOptions { SkipInvalidRows = true, AllowUnknownFields = true, SuppressInsertErrors = true } };
154+
int[] bothRowsIndexes = new int[] { 0, 1 };
155+
int[] oneRowIndex = new int[] { 1 };
156+
157+
yield return new object[] { new InsertOptions { SuppressInsertErrors = true }, bothRowsIndexes };
158+
yield return new object[] { new InsertOptions { SkipInvalidRows = false, SuppressInsertErrors = true }, bothRowsIndexes };
159+
yield return new object[] { new InsertOptions { SkipInvalidRows = true, SuppressInsertErrors = true }, bothRowsIndexes };
160+
yield return new object[] { new InsertOptions { AllowUnknownFields = false, SuppressInsertErrors = true }, bothRowsIndexes };
161+
yield return new object[] { new InsertOptions { AllowUnknownFields = true, SuppressInsertErrors = true }, bothRowsIndexes };
162+
yield return new object[] { new InsertOptions { SkipInvalidRows = false, AllowUnknownFields = false, SuppressInsertErrors = true }, bothRowsIndexes };
163+
yield return new object[] { new InsertOptions { SkipInvalidRows = false, AllowUnknownFields = true, SuppressInsertErrors = true }, bothRowsIndexes };
164+
yield return new object[] { new InsertOptions { SkipInvalidRows = true, AllowUnknownFields = false, SuppressInsertErrors = true }, bothRowsIndexes };
165+
yield return new object[] { new InsertOptions { SkipInvalidRows = true, AllowUnknownFields = true, SuppressInsertErrors = true }, oneRowIndex };
142166
}
143167
}
144168

145169
[Theory]
146170
[MemberData(nameof(BadDataSilentOptions))]
147-
public void InsertRow_BadData_Silent(InsertOptions options)
171+
public void InsertRow_BadData_Silent(InsertOptions options, int[] errorRowsIndexes)
148172
{
149173
var client = BigQueryClient.Create(_fixture.ProjectId);
150174
var dataset = client.GetDataset(_fixture.DatasetId);
@@ -157,7 +181,14 @@ public void InsertRow_BadData_Silent(InsertOptions options)
157181
new BigQueryInsertRow { { "noSuchField", 10 } },
158182
new BigQueryInsertRow { {"year", "Unknown"} }
159183
};
160-
table.InsertRows(rows, options);
184+
var insertResult = table.InsertRows(rows, options);
185+
186+
Assert.Equal(errorRowsIndexes.Length, insertResult.OriginalRowsWithErrors);
187+
Assert.Equal(errorRowsIndexes,
188+
insertResult.Errors.
189+
Select(e => (int?) e.OriginalRowIndex ?? -1).
190+
OrderBy(index => index).
191+
ToArray());
161192
}
162193

163194
[Fact]
@@ -185,7 +216,9 @@ public void InsertRow_BadData_IgnoreExtraColumn()
185216
// Even though SuppressInsertErrors is false, this won't throw
186217
// because server side, unknown fields are ignored silently.
187218
var options = new InsertOptions { AllowUnknownFields = true };
188-
_fixture.InsertAndWait(table, () => table.InsertRow(row, options), 1);
219+
var insertResult = _fixture.InsertAndWait(table, () => table.InsertRow(row, options), 1);
220+
221+
AssertAllRowsInserted(insertResult);
189222
}
190223

191224
[Fact]
@@ -207,7 +240,13 @@ public void InsertRow_BadData_IgnoreBadRows_Silent()
207240

208241
var options = new InsertOptions { SkipInvalidRows = true, SuppressInsertErrors = true};
209242
// Only one row inserted, we are not ignoring unknown fields so the last two rows are bad.
210-
_fixture.InsertAndWait(table, () => table.InsertRows(rows, options), 1);
243+
var insertResult = _fixture.InsertAndWait(table, () => table.InsertRows(rows, options), 1);
244+
245+
Assert.Equal(2, insertResult.OriginalRowsWithErrors);
246+
Assert.Equal(3, insertResult.InsertAttemptRowCount);
247+
Assert.Equal(BigQueryInsertStatus.SomeRowsInserted, insertResult.Status);
248+
Assert.Contains(insertResult.Errors, e => e.OriginalRowIndex == 1);
249+
Assert.Contains(insertResult.Errors, e => e.OriginalRowIndex == 2);
211250
}
212251

213252
[Fact]
@@ -230,8 +269,8 @@ public void InsertRow_BadData_IgnoreBadRows_Throws()
230269
var options = new InsertOptions { SkipInvalidRows = true, SuppressInsertErrors = false };
231270
var exception = Assert.Throws<GoogleApiException>(() => table.InsertRows(rows, options));
232271
Assert.Equal(2, exception.Error.Errors.Count);
233-
Assert.Contains(exception.Error.Errors, e => e.Message.Contains("Row 1"));
234-
Assert.Contains(exception.Error.Errors, e => e.Message.Contains("Row 2"));
272+
Assert.Contains(exception.Error.Errors, e => e.Message.ToLower().Contains("in row 1"));
273+
Assert.Contains(exception.Error.Errors, e => e.Message.ToLower().Contains("in row 2"));
235274
}
236275

237276
[Fact]
@@ -253,7 +292,12 @@ public void InsertRow_BadData_IgnoreUnknownAndBadRows_Silent()
253292

254293
var options = new InsertOptions { AllowUnknownFields = true, SkipInvalidRows = true, SuppressInsertErrors = true };
255294
// Now two rows are inserted, we are ignoring unknown fields so only the last row is bad.
256-
_fixture.InsertAndWait(table, () => table.InsertRows(rows, options), 2);
295+
var insertResult = _fixture.InsertAndWait(table, () => table.InsertRows(rows, options), 2);
296+
297+
Assert.Equal(1, insertResult.OriginalRowsWithErrors);
298+
Assert.Equal(3, insertResult.InsertAttemptRowCount);
299+
Assert.Equal(BigQueryInsertStatus.SomeRowsInserted, insertResult.Status);
300+
Assert.Contains(insertResult.Errors, e => e.OriginalRowIndex == 2);
257301
}
258302

259303
[Fact]
@@ -276,7 +320,7 @@ public void InsertRow_BadData_IgnoreUnknownAndBadRows_Throws()
276320
var options = new InsertOptions { AllowUnknownFields = true, SkipInvalidRows = true, SuppressInsertErrors = false };
277321
var exception = Assert.Throws<GoogleApiException>(() => table.InsertRows(rows, options));
278322
Assert.Equal(1, exception.Error.Errors.Count);
279-
Assert.Contains("Row 2", exception.Error.Errors[0].Message);
323+
Assert.Contains("in row 2", exception.Error.Errors[0].Message, StringComparison.InvariantCultureIgnoreCase);
280324
}
281325

282326
[Fact]

‎apis/Google.Cloud.BigQuery.V2/Google.Cloud.BigQuery.V2.Tests/BigQueryClientTest.cs‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -657,7 +657,7 @@ public void InsertEquivalents_SingleRow()
657657
var options = new InsertOptions();
658658
var stream = new MemoryStream();
659659
var row = new BigQueryInsertRow();
660-
VerifyEquivalent(
660+
VerifyEquivalent(new BigQueryInsertResults(new DerivedBigQueryClient(), options, Enumerable.Repeat(row, 1).ToList(), new TableDataInsertAllResponse()),
661661
client => client.InsertRows(MatchesWhenSerialized(reference), new[] { row }, options),
662662
client => client.InsertRow(datasetId, tableId, row, options),
663663
client => client.InsertRow(ProjectId, datasetId, tableId, row, options),
@@ -674,7 +674,7 @@ public void InsertEquivalents_RowCollection()
674674
var options = new InsertOptions();
675675
var stream = new MemoryStream();
676676
var rows = new[] { new BigQueryInsertRow(), new BigQueryInsertRow() };
677-
VerifyEquivalent(
677+
VerifyEquivalent(new BigQueryInsertResults(new DerivedBigQueryClient(), options, rows, new TableDataInsertAllResponse()),
678678
client => client.InsertRows(MatchesWhenSerialized(reference), rows, options),
679679
client => client.InsertRows(datasetId, tableId, rows, options),
680680
client => client.InsertRows(ProjectId, datasetId, tableId, rows, options),
@@ -691,7 +691,7 @@ public void InsertEquivalents_ParamsRows()
691691
var options = new InsertOptions();
692692
var stream = new MemoryStream();
693693
var rows = new[] { new BigQueryInsertRow(), new BigQueryInsertRow() };
694-
VerifyEquivalent(
694+
VerifyEquivalent(new BigQueryInsertResults(new DerivedBigQueryClient(), options, rows, new TableDataInsertAllResponse()),
695695
client => client.InsertRows(MatchesWhenSerialized(reference), rows, null),
696696
client => client.InsertRows(datasetId, tableId, rows[0], rows[1]),
697697
client => client.InsertRows(ProjectId, datasetId, tableId, rows[0], rows[1]),
@@ -1205,7 +1205,7 @@ public void InsertAsyncEquivalents_SingleRow()
12051205
var token = new CancellationTokenSource().Token;
12061206
var stream = new MemoryStream();
12071207
var row = new BigQueryInsertRow();
1208-
VerifyEquivalentAsync(
1208+
VerifyEquivalentAsync(new BigQueryInsertResults(new DerivedBigQueryClient(), options, Enumerable.Repeat(row, 1).ToList(), new TableDataInsertAllResponse()),
12091209
client => client.InsertRowsAsync(MatchesWhenSerialized(reference), new[] { row }, options, token),
12101210
client => client.InsertRowAsync(datasetId, tableId, row, options, token),
12111211
client => client.InsertRowAsync(ProjectId, datasetId, tableId, row, options, token),
@@ -1223,7 +1223,7 @@ public void InsertAsyncEquivalents_RowCollection()
12231223
var token = new CancellationTokenSource().Token;
12241224
var stream = new MemoryStream();
12251225
var rows = new[] { new BigQueryInsertRow(), new BigQueryInsertRow() };
1226-
VerifyEquivalentAsync(
1226+
VerifyEquivalentAsync(new BigQueryInsertResults(new DerivedBigQueryClient(), options, rows, new TableDataInsertAllResponse()),
12271227
client => client.InsertRowsAsync(MatchesWhenSerialized(reference), rows, options, token),
12281228
client => client.InsertRowsAsync(datasetId, tableId, rows, options, token),
12291229
client => client.InsertRowsAsync(ProjectId, datasetId, tableId, rows, options, token),
@@ -1241,7 +1241,7 @@ public void InsertAsyncEquivalents_ParamsRows()
12411241
var token = new CancellationTokenSource().Token;
12421242
var stream = new MemoryStream();
12431243
var rows = new[] { new BigQueryInsertRow(), new BigQueryInsertRow() };
1244-
VerifyEquivalentAsync(
1244+
VerifyEquivalentAsync(new BigQueryInsertResults(new DerivedBigQueryClient(), options, rows, new TableDataInsertAllResponse()),
12451245
client => client.InsertRowsAsync(MatchesWhenSerialized(reference), rows, null, default),
12461246
client => client.InsertRowsAsync(datasetId, tableId, rows[0], rows[1]),
12471247
client => client.InsertRowsAsync(ProjectId, datasetId, tableId, rows[0], rows[1]),

0 commit comments

Comments
 (0)