diff --git a/google-cloud-bigquery/pom.xml b/google-cloud-bigquery/pom.xml index 6e226056ba..37acecdeb1 100644 --- a/google-cloud-bigquery/pom.xml +++ b/google-cloud-bigquery/pom.xml @@ -15,7 +15,6 @@ google-cloud-bigquery - 4.1.119.Final @@ -117,37 +116,6 @@ org.apache.arrow arrow-memory-netty - - - - io.netty - netty-common - - - io.netty - netty-buffer - - - runtime - - - - io.netty - netty-common - ${netty.version} - runtime - - - io.netty - netty-buffer - ${netty.version} - runtime @@ -290,6 +258,7 @@ true -J--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED + -J--add-opens=java.base/java.nio=java-base,ALL-UNNAMED diff --git a/google-cloud-bigquery/src/main/resources/META-INF/native-image/com.google.cloud/google-cloud-bigquery/native-image.properties b/google-cloud-bigquery/src/main/resources/META-INF/native-image/com.google.cloud/google-cloud-bigquery/native-image.properties index f7cce6dec2..10d4436d26 100644 --- a/google-cloud-bigquery/src/main/resources/META-INF/native-image/com.google.cloud/google-cloud-bigquery/native-image.properties +++ b/google-cloud-bigquery/src/main/resources/META-INF/native-image/com.google.cloud/google-cloud-bigquery/native-image.properties @@ -1 +1,6 @@ -Args = --add-opens=java.base/java.nio=ALL-UNNAMED \ No newline at end of file +Args = --add-opens=java.base/java.nio=ALL-UNNAMED \ + + #--initialize-at-build-time=org.apache.arrow.memory.BaseAllocator \ + #--initialize-at-build-time=org.apache.arrow.memory.BaseAllocator$Config \ + #--initialize-at-build-time=org.apache.arrow.memory.DefaultAllocationManagerOption \ + #--initialize-at-build-time=org.apache.arrow.memory.netty.NettyAllocationManager$1 \ No newline at end of file diff --git a/google-cloud-bigquery/src/main/resources/META-INF/native-image/com.google.cloud/google-cloud-bigquery/reflect-config.json b/google-cloud-bigquery/src/main/resources/META-INF/native-image/com.google.cloud/google-cloud-bigquery/reflect-config.json index 8f6d928da4..54f58a5221 100644 --- a/google-cloud-bigquery/src/main/resources/META-INF/native-image/com.google.cloud/google-cloud-bigquery/reflect-config.json +++ b/google-cloud-bigquery/src/main/resources/META-INF/native-image/com.google.cloud/google-cloud-bigquery/reflect-config.json @@ -29,9 +29,48 @@ "name":"java.nio.DirectByteBuffer", "methods":[{"name":"","parameterTypes":["long","int"] }] }, + + { + "name":"org.apache.arrow.memory.BaseAllocator", + "allDeclaredFields":true, + "queryAllDeclaredFields":true, + "queryAllDeclaredMethods":true, + "queryAllDeclaredConstructors":true + }, + { + "name":"org.apache.arrow.memory.BaseAllocator$Config", + "allDeclaredFields":true, + "queryAllDeclaredFields":true, + "queryAllDeclaredMethods":true, + "queryAllDeclaredConstructors":true + }, + { + "name":"org.apache.arrow.memory.DefaultAllocationManagerOption", + "allDeclaredFields":true, + "queryAllDeclaredFields":true, + "queryAllDeclaredMethods":true, + "queryAllDeclaredConstructors":true + }, + { + "name":"org.apache.arrow.memory.netty.NettyAllocationManager$1", + "allDeclaredFields":true, + "queryAllDeclaredFields":true, + "queryAllDeclaredMethods":true, + "queryAllDeclaredConstructors":true + }, + { + "name":"org.apache.arrow.memory.netty.DefaultAllocationManagerFactory", + "allDeclaredFields":true, + "queryAllDeclaredFields":true, + "queryAllDeclaredMethods":true, + "queryAllDeclaredConstructors":true + }, { "name":"org.apache.arrow.memory.DefaultAllocationManagerFactory", - "fields":[{"name":"FACTORY"}] + "allDeclaredFields":true, + "queryAllDeclaredFields":true, + "queryAllDeclaredMethods":true, + "queryAllDeclaredConstructors":true }, { "name":"org.apache.arrow.vector.types.pojo.ArrowType", @@ -50,6 +89,18 @@ "allDeclaredFields":true, "queryAllDeclaredMethods":true }, + { + "name": "org.apache.arrow.vector.types.pojo.DictionaryEncoding", + "allDeclaredFields": true + }, + { + "name": "org.apache.arrow.vector.types.pojo.Field", + "allDeclaredFields": true + }, + { + "name": "org.apache.arrow.vector.types.pojo.Schema", + "allDeclaredFields": true + }, { "name":"io.netty.buffer.AbstractReferenceCountedByteBuf", "fields":[{"name":"refCnt"}] diff --git a/google-cloud-bigquery/src/main/resources/META-INF/native-image/com.google.cloud/google-cloud-bigquery/resource-config.json b/google-cloud-bigquery/src/main/resources/META-INF/native-image/com.google.cloud/google-cloud-bigquery/resource-config.json index dfdcc58c44..adb396c74e 100644 --- a/google-cloud-bigquery/src/main/resources/META-INF/native-image/com.google.cloud/google-cloud-bigquery/resource-config.json +++ b/google-cloud-bigquery/src/main/resources/META-INF/native-image/com.google.cloud/google-cloud-bigquery/resource-config.json @@ -1,6 +1,20 @@ { "resources":{ - "includes":[{ - "pattern":"\\Qorg/apache/arrow/memory/DefaultAllocationManagerFactory.class\\E" - }]} + "includes":[ + { + "pattern":"\\Qorg/apache/arrow/memory/DefaultAllocationManagerFactory.class\\E" + } + ] + }, + "globs":[ + { + "glob": "org/apache/arrow/memory/DefaultAllocationManagerFactory.class" + }, + { + "glob": "org/apache/arrow/memory/netty/DefaultAllocationManagerFactory.class" + }, + { + "glob": "org/apache/arrow/memory/unsafe/DefaultAllocationManagerFactory.class" + } + ] } \ No newline at end of file diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index c9f6296cc5..1b54cee060 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -214,7 +214,23 @@ import org.junit.rules.Timeout; import org.threeten.extra.PeriodDuration; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; public class ITBigQueryTest { + @Test + public void testArrowForName() throws Exception { + // mvn test -Pnative '-Dtest=com.google.cloud.bigquery.it.ITBigQueryTest#testArrowForName' -DtrimStackTrace=false + final String clazzName = "org.apache.arrow.memory.netty.DefaultAllocationManagerFactory"; + try { + BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + //java.lang.reflect.Field field = Class.forName(clazzName).getDeclaredField("FACTORY"); + //field.setAccessible(true); + //System.out.println("OK"); + } catch (Exception e) { + throw new RuntimeException("Unable to instantiate Allocation Manager for " + clazzName, e); + } + } + private static final byte[] BYTES = {0xD, 0xE, 0xA, 0xD}; private static final String BYTES_BASE64 = BaseEncoding.base64().encode(BYTES); @@ -1064,118 +1080,6 @@ public CompletableResultCode shutdown() { @Rule public Timeout globalTimeout = Timeout.seconds(300); - @BeforeClass - public static void beforeClass() throws InterruptedException, IOException { - RemoteBigQueryHelper bigqueryHelper = RemoteBigQueryHelper.create(); - RemoteStorageHelper storageHelper = RemoteStorageHelper.create(); - Map labels = ImmutableMap.of("test-job-name", "test-load-job"); - SdkTracerProvider tracerProvider = - SdkTracerProvider.builder() - .addSpanProcessor(SimpleSpanProcessor.create(new TestSpanExporter())) - .setSampler(Sampler.alwaysOn()) - .build(); - otel = OpenTelemetrySdk.builder().setTracerProvider(tracerProvider).buildAndRegisterGlobal(); - - bigquery = bigqueryHelper.getOptions().getService(); - storage = storageHelper.getOptions().getService(); - storage.create(BucketInfo.of(BUCKET)); - storage.create( - BlobInfo.newBuilder(BUCKET, LOAD_FILE).setContentType("text/plain").build(), - CSV_CONTENT.getBytes(StandardCharsets.UTF_8)); - storage.create( - BlobInfo.newBuilder(BUCKET, LOAD_FILE_NULL).setContentType("text/plain").build(), - CSV_CONTENT_NULL.getBytes(StandardCharsets.UTF_8)); - storage.create( - BlobInfo.newBuilder(BUCKET, LOAD_FILE_FLEXIBLE_COLUMN_NAME) - .setContentType("text/plain") - .build(), - CSV_CONTENT_FLEXIBLE_COLUMN.getBytes(StandardCharsets.UTF_8)); - storage.create( - BlobInfo.newBuilder(BUCKET, JSON_LOAD_FILE).setContentType("application/json").build(), - JSON_CONTENT.getBytes(StandardCharsets.UTF_8)); - storage.create( - BlobInfo.newBuilder(BUCKET, JSON_LOAD_FILE_SIMPLE) - .setContentType("application/json") - .build(), - JSON_CONTENT_SIMPLE.getBytes(StandardCharsets.UTF_8)); - InputStream stream = - ITBigQueryTest.class.getClassLoader().getResourceAsStream("QueryTestData.csv"); - storage.createFrom( - BlobInfo.newBuilder(BUCKET, LOAD_FILE_LARGE).setContentType("text/plain").build(), stream); - storage.create( - BlobInfo.newBuilder(BUCKET, JSON_LOAD_FILE_BQ_RESULTSET) - .setContentType("application/json") - .build(), - JSON_CONTENT_BQ_RESULTSET.getBytes(StandardCharsets.UTF_8)); - DatasetInfo info = - DatasetInfo.newBuilder(DATASET).setDescription(DESCRIPTION).setLabels(LABELS).build(); - bigquery.create(info); - DatasetInfo info2 = - DatasetInfo.newBuilder(MODEL_DATASET).setDescription("java model lifecycle").build(); - bigquery.create(info2); - DatasetInfo info3 = - DatasetInfo.newBuilder(ROUTINE_DATASET).setDescription("java routine lifecycle").build(); - bigquery.create(info3); - - LoadJobConfiguration configuration = - LoadJobConfiguration.newBuilder( - TABLE_ID, "gs://" + BUCKET + "/" + JSON_LOAD_FILE, FormatOptions.json()) - .setCreateDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED) - .setSchema(TABLE_SCHEMA) - .setLabels(labels) - .build(); - Job job = bigquery.create(JobInfo.of(configuration)); - job = job.waitFor(); - assertNull(job.getStatus().getError()); - LoadJobConfiguration loadJobConfiguration = job.getConfiguration(); - assertEquals(labels, loadJobConfiguration.getLabels()); - - LoadJobConfiguration configurationFastQuery = - LoadJobConfiguration.newBuilder( - TABLE_ID_FASTQUERY, "gs://" + BUCKET + "/" + JSON_LOAD_FILE, FormatOptions.json()) - .setCreateDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED) - .setSchema(TABLE_SCHEMA) - .setLabels(labels) - .build(); - Job jobFastQuery = bigquery.create(JobInfo.of(configurationFastQuery)); - jobFastQuery = jobFastQuery.waitFor(); - assertNull(jobFastQuery.getStatus().getError()); - - LoadJobConfiguration configFastQueryBQResultset = - LoadJobConfiguration.newBuilder( - TABLE_ID_FASTQUERY_BQ_RESULTSET, - "gs://" + BUCKET + "/" + JSON_LOAD_FILE_BQ_RESULTSET, - FormatOptions.json()) - .setCreateDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED) - .setSchema(BQ_RESULTSET_SCHEMA) - .setLabels(labels) - .build(); - Job jobFastQueryBQResultSet = bigquery.create(JobInfo.of(configFastQueryBQResultset)); - jobFastQueryBQResultSet = jobFastQueryBQResultSet.waitFor(); - assertNull(jobFastQueryBQResultSet.getStatus().getError()); - - LoadJobConfiguration configurationDDL = - LoadJobConfiguration.newBuilder( - TABLE_ID_DDL, "gs://" + BUCKET + "/" + JSON_LOAD_FILE_SIMPLE, FormatOptions.json()) - .setCreateDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED) - .setSchema(DDL_TABLE_SCHEMA) - .setLabels(labels) - .build(); - Job jobDDL = bigquery.create(JobInfo.of(configurationDDL)); - jobDDL = jobDDL.waitFor(); - assertNull(jobDDL.getStatus().getError()); - - LoadJobConfiguration configurationLargeTable = - LoadJobConfiguration.newBuilder( - TABLE_ID_LARGE, "gs://" + BUCKET + "/" + LOAD_FILE_LARGE, FormatOptions.csv()) - .setCreateDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED) - .setSchema(LARGE_TABLE_SCHEMA) - .setLabels(labels) - .build(); - Job jobLargeTable = bigquery.create(JobInfo.of(configurationLargeTable)); - jobLargeTable = jobLargeTable.waitFor(); - assertNull(jobLargeTable.getStatus().getError()); - } @AfterClass public static void afterClass() throws ExecutionException, InterruptedException { diff --git a/google-cloud-bigquery/src/test/resources/META-INF/native-image/com.google.cloud/google-cloud-bigquery/native-image.properties b/google-cloud-bigquery/src/test/resources/META-INF/native-image/com.google.cloud/google-cloud-bigquery/native-image.properties new file mode 100644 index 0000000000..7c1e68d363 --- /dev/null +++ b/google-cloud-bigquery/src/test/resources/META-INF/native-image/com.google.cloud/google-cloud-bigquery/native-image.properties @@ -0,0 +1,3 @@ +Args = --initialize-at-build-time=org.junit +#Args = --initialize-at-build-time=org.mockito \ +#--initialize-at-build-time=net.bytebuddy.TypeCache$WithInlineExpunction