debug: repro of GraalVM problem with arrow 17'

google-cloud-bigquery/pom.xml

@@ -15,7 +15,6 @@
   </parent>
   <properties>
     <site.installationModule>google-cloud-bigquery</site.installationModule>
-    <netty.version>4.1.119.Final</netty.version>
   </properties>
   <dependencies>
     <dependency>
@@ -117,37 +116,6 @@
     <dependency>
       <groupId>org.apache.arrow</groupId>
       <artifactId>arrow-memory-netty</artifactId>
-      <!--
-      Due to JDK8 requirement, we are forced to use arrow version below v18
-      which has io.netty:netty-common a dep. and its vulnerabilities
-      -->
-      <exclusions>
-        <exclusion>
-          <groupId>io.netty</groupId>
-          <artifactId>netty-common</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>io.netty</groupId>
-          <artifactId>netty-buffer</artifactId>
-        </exclusion>
-      </exclusions>
-      <scope>runtime</scope>
-    </dependency>
-    <!--
-    Define io.netty versions to override org.apache.arrow:arrow-memory-netty
-    transitive dependency versions which contains vulnerabilities.
-    -->
-    <dependency>
-      <groupId>io.netty</groupId>
-      <artifactId>netty-common</artifactId>
-      <version>${netty.version}</version>
-      <scope>runtime</scope>
-    </dependency>
-    <dependency>
-      <groupId>io.netty</groupId>
-      <artifactId>netty-buffer</artifactId>
-      <version>${netty.version}</version>
-      <scope>runtime</scope>
     </dependency>
 
     <!-- auto-value creates a class that uses an annotation from error_prone_annotations -->
@@ -290,6 +258,7 @@
               <fork>true</fork>
               <compilerArgs>
                 <arg>-J--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED</arg>
+                <arg>-J--add-opens=java.base/java.nio=java-base,ALL-UNNAMED</arg>
               </compilerArgs>
             </configuration>
           </plugin>

google-cloud-bigquery/src/main/resources/META-INF/native-image/com.google.cloud/google-cloud-bigquery/native-image.properties

@@ -1 +1,6 @@
-Args = --add-opens=java.base/java.nio=ALL-UNNAMED
+Args = --add-opens=java.base/java.nio=ALL-UNNAMED \
+
+ #--initialize-at-build-time=org.apache.arrow.memory.BaseAllocator \
+ #--initialize-at-build-time=org.apache.arrow.memory.BaseAllocator$Config \
+ #--initialize-at-build-time=org.apache.arrow.memory.DefaultAllocationManagerOption \
+ #--initialize-at-build-time=org.apache.arrow.memory.netty.NettyAllocationManager$1

google-cloud-bigquery/src/main/resources/META-INF/native-image/com.google.cloud/google-cloud-bigquery/reflect-config.json

@@ -29,9 +29,48 @@
     "name":"java.nio.DirectByteBuffer",
     "methods":[{"name":"<init>","parameterTypes":["long","int"] }]
   },
+
+  {
+    "name":"org.apache.arrow.memory.BaseAllocator",
+    "allDeclaredFields":true,
+    "queryAllDeclaredFields":true,
+    "queryAllDeclaredMethods":true,
+    "queryAllDeclaredConstructors":true
+  },
+  {
+    "name":"org.apache.arrow.memory.BaseAllocator$Config",
+    "allDeclaredFields":true,
+    "queryAllDeclaredFields":true,
+    "queryAllDeclaredMethods":true,
+    "queryAllDeclaredConstructors":true
+  },
+  {
+    "name":"org.apache.arrow.memory.DefaultAllocationManagerOption",
+    "allDeclaredFields":true,
+    "queryAllDeclaredFields":true,
+    "queryAllDeclaredMethods":true,
+    "queryAllDeclaredConstructors":true
+  },
+  {
+    "name":"org.apache.arrow.memory.netty.NettyAllocationManager$1",
+    "allDeclaredFields":true,
+    "queryAllDeclaredFields":true,
+    "queryAllDeclaredMethods":true,
+    "queryAllDeclaredConstructors":true
+  },
+  {
+    "name":"org.apache.arrow.memory.netty.DefaultAllocationManagerFactory",
+    "allDeclaredFields":true,
+    "queryAllDeclaredFields":true,
+    "queryAllDeclaredMethods":true,
+    "queryAllDeclaredConstructors":true
+  },
   {
     "name":"org.apache.arrow.memory.DefaultAllocationManagerFactory",
-    "fields":[{"name":"FACTORY"}]
+    "allDeclaredFields":true,
+    "queryAllDeclaredFields":true,
+    "queryAllDeclaredMethods":true,
+    "queryAllDeclaredConstructors":true
   },
   {
     "name":"org.apache.arrow.vector.types.pojo.ArrowType",
@@ -50,6 +89,18 @@
     "allDeclaredFields":true,
     "queryAllDeclaredMethods":true
   },
+  {
+    "name": "org.apache.arrow.vector.types.pojo.DictionaryEncoding",
+    "allDeclaredFields": true
+  },
+  {
+    "name": "org.apache.arrow.vector.types.pojo.Field",
+    "allDeclaredFields": true
+  },
+  {
+    "name": "org.apache.arrow.vector.types.pojo.Schema",
+    "allDeclaredFields": true
+  },
   {
     "name":"io.netty.buffer.AbstractReferenceCountedByteBuf",
     "fields":[{"name":"refCnt"}]

google-cloud-bigquery/src/main/resources/META-INF/native-image/com.google.cloud/google-cloud-bigquery/resource-config.json

@@ -1,6 +1,20 @@
 {
   "resources":{
-    "includes":[{
-      "pattern":"\\Qorg/apache/arrow/memory/DefaultAllocationManagerFactory.class\\E"
-    }]}
+    "includes":[
+      {
+        "pattern":"\\Qorg/apache/arrow/memory/DefaultAllocationManagerFactory.class\\E"
+      }
+    ]
+  },
+  "globs":[
+      {
+        "glob": "org/apache/arrow/memory/DefaultAllocationManagerFactory.class"
+      },
+      {
+        "glob": "org/apache/arrow/memory/netty/DefaultAllocationManagerFactory.class"
+      },
+      {
+        "glob": "org/apache/arrow/memory/unsafe/DefaultAllocationManagerFactory.class"
+      }
+   ]
 }

google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java

@@ -214,7 +214,23 @@
 import org.junit.rules.Timeout;
 import org.threeten.extra.PeriodDuration;
 
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
 public class ITBigQueryTest {
+  @Test
+  public void testArrowForName() throws Exception {
+    // mvn test -Pnative '-Dtest=com.google.cloud.bigquery.it.ITBigQueryTest#testArrowForName' -DtrimStackTrace=false
+    final String clazzName = "org.apache.arrow.memory.netty.DefaultAllocationManagerFactory";
+    try {
+      BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
+      //java.lang.reflect.Field field = Class.forName(clazzName).getDeclaredField("FACTORY");
+      //field.setAccessible(true);
+      //System.out.println("OK");
+    } catch (Exception e) {
+      throw new RuntimeException("Unable to instantiate Allocation Manager for " + clazzName, e);
+    }
+  }
+
 
   private static final byte[] BYTES = {0xD, 0xE, 0xA, 0xD};
   private static final String BYTES_BASE64 = BaseEncoding.base64().encode(BYTES);
@@ -1064,118 +1080,6 @@ public CompletableResultCode shutdown() {
 
   @Rule public Timeout globalTimeout = Timeout.seconds(300);
 
-  @BeforeClass
-  public static void beforeClass() throws InterruptedException, IOException {
-    RemoteBigQueryHelper bigqueryHelper = RemoteBigQueryHelper.create();
-    RemoteStorageHelper storageHelper = RemoteStorageHelper.create();
-    Map<String, String> labels = ImmutableMap.of("test-job-name", "test-load-job");
-    SdkTracerProvider tracerProvider =
-        SdkTracerProvider.builder()
-            .addSpanProcessor(SimpleSpanProcessor.create(new TestSpanExporter()))
-            .setSampler(Sampler.alwaysOn())
-            .build();
-    otel = OpenTelemetrySdk.builder().setTracerProvider(tracerProvider).buildAndRegisterGlobal();
-
-    bigquery = bigqueryHelper.getOptions().getService();
-    storage = storageHelper.getOptions().getService();
-    storage.create(BucketInfo.of(BUCKET));
-    storage.create(
-        BlobInfo.newBuilder(BUCKET, LOAD_FILE).setContentType("text/plain").build(),
-        CSV_CONTENT.getBytes(StandardCharsets.UTF_8));
-    storage.create(
-        BlobInfo.newBuilder(BUCKET, LOAD_FILE_NULL).setContentType("text/plain").build(),
-        CSV_CONTENT_NULL.getBytes(StandardCharsets.UTF_8));
-    storage.create(
-        BlobInfo.newBuilder(BUCKET, LOAD_FILE_FLEXIBLE_COLUMN_NAME)
-            .setContentType("text/plain")
-            .build(),
-        CSV_CONTENT_FLEXIBLE_COLUMN.getBytes(StandardCharsets.UTF_8));
-    storage.create(
-        BlobInfo.newBuilder(BUCKET, JSON_LOAD_FILE).setContentType("application/json").build(),
-        JSON_CONTENT.getBytes(StandardCharsets.UTF_8));
-    storage.create(
-        BlobInfo.newBuilder(BUCKET, JSON_LOAD_FILE_SIMPLE)
-            .setContentType("application/json")
-            .build(),
-        JSON_CONTENT_SIMPLE.getBytes(StandardCharsets.UTF_8));
-    InputStream stream =
-        ITBigQueryTest.class.getClassLoader().getResourceAsStream("QueryTestData.csv");
-    storage.createFrom(
-        BlobInfo.newBuilder(BUCKET, LOAD_FILE_LARGE).setContentType("text/plain").build(), stream);
-    storage.create(
-        BlobInfo.newBuilder(BUCKET, JSON_LOAD_FILE_BQ_RESULTSET)
-            .setContentType("application/json")
-            .build(),
-        JSON_CONTENT_BQ_RESULTSET.getBytes(StandardCharsets.UTF_8));
-    DatasetInfo info =
-        DatasetInfo.newBuilder(DATASET).setDescription(DESCRIPTION).setLabels(LABELS).build();
-    bigquery.create(info);
-    DatasetInfo info2 =
-        DatasetInfo.newBuilder(MODEL_DATASET).setDescription("java model lifecycle").build();
-    bigquery.create(info2);
-    DatasetInfo info3 =
-        DatasetInfo.newBuilder(ROUTINE_DATASET).setDescription("java routine lifecycle").build();
-    bigquery.create(info3);
-
-    LoadJobConfiguration configuration =
-        LoadJobConfiguration.newBuilder(
-                TABLE_ID, "gs://" + BUCKET + "/" + JSON_LOAD_FILE, FormatOptions.json())
-            .setCreateDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED)
-            .setSchema(TABLE_SCHEMA)
-            .setLabels(labels)
-            .build();
-    Job job = bigquery.create(JobInfo.of(configuration));
-    job = job.waitFor();
-    assertNull(job.getStatus().getError());
-    LoadJobConfiguration loadJobConfiguration = job.getConfiguration();
-    assertEquals(labels, loadJobConfiguration.getLabels());
-
-    LoadJobConfiguration configurationFastQuery =
-        LoadJobConfiguration.newBuilder(
-                TABLE_ID_FASTQUERY, "gs://" + BUCKET + "/" + JSON_LOAD_FILE, FormatOptions.json())
-            .setCreateDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED)
-            .setSchema(TABLE_SCHEMA)
-            .setLabels(labels)
-            .build();
-    Job jobFastQuery = bigquery.create(JobInfo.of(configurationFastQuery));
-    jobFastQuery = jobFastQuery.waitFor();
-    assertNull(jobFastQuery.getStatus().getError());
-
-    LoadJobConfiguration configFastQueryBQResultset =
-        LoadJobConfiguration.newBuilder(
-                TABLE_ID_FASTQUERY_BQ_RESULTSET,
-                "gs://" + BUCKET + "/" + JSON_LOAD_FILE_BQ_RESULTSET,
-                FormatOptions.json())
-            .setCreateDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED)
-            .setSchema(BQ_RESULTSET_SCHEMA)
-            .setLabels(labels)
-            .build();
-    Job jobFastQueryBQResultSet = bigquery.create(JobInfo.of(configFastQueryBQResultset));
-    jobFastQueryBQResultSet = jobFastQueryBQResultSet.waitFor();
-    assertNull(jobFastQueryBQResultSet.getStatus().getError());
-
-    LoadJobConfiguration configurationDDL =
-        LoadJobConfiguration.newBuilder(
-                TABLE_ID_DDL, "gs://" + BUCKET + "/" + JSON_LOAD_FILE_SIMPLE, FormatOptions.json())
-            .setCreateDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED)
-            .setSchema(DDL_TABLE_SCHEMA)
-            .setLabels(labels)
-            .build();
-    Job jobDDL = bigquery.create(JobInfo.of(configurationDDL));
-    jobDDL = jobDDL.waitFor();
-    assertNull(jobDDL.getStatus().getError());
-
-    LoadJobConfiguration configurationLargeTable =
-        LoadJobConfiguration.newBuilder(
-                TABLE_ID_LARGE, "gs://" + BUCKET + "/" + LOAD_FILE_LARGE, FormatOptions.csv())
-            .setCreateDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED)
-            .setSchema(LARGE_TABLE_SCHEMA)
-            .setLabels(labels)
-            .build();
-    Job jobLargeTable = bigquery.create(JobInfo.of(configurationLargeTable));
-    jobLargeTable = jobLargeTable.waitFor();
-    assertNull(jobLargeTable.getStatus().getError());
-  }
 
   @AfterClass
   public static void afterClass() throws ExecutionException, InterruptedException {

google-cloud-bigquery/src/test/resources/META-INF/native-image/com.google.cloud/google-cloud-bigquery/native-image.properties

@@ -0,0 +1,3 @@
+Args = --initialize-at-build-time=org.junit
+#Args = --initialize-at-build-time=org.mockito \
+#--initialize-at-build-time=net.bytebuddy.TypeCache$WithInlineExpunction