Add support for Apache Arrow C data interface (#13)

fix #9
datafusion-contrib · Jul 5, 2022 · db57572 · db57572
1 parent 8b0882c
commit db57572
Show file tree

Hide file tree

Showing 17 changed files with 603 additions and 113 deletions.
diff --git a/.editorconfig b/.editorconfig
@@ -24,7 +24,7 @@ end_of_line = lf
 insert_final_newline = true
 trim_trailing_whitespace = true
 
-[*.{vala,rust}]
+[*.{vala,rs}]
 indent_style = space
 end_of_line = lf
 insert_final_newline = true

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -93,128 +93,121 @@ jobs:
     runs-on: ${{ matrix.runs-on }}
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-python@v3
+      - uses: actions/setup-python@v4
         if: |
           matrix.runs-on != 'windows-latest'
         with:
           python-version: '3.x'
-      - uses: ruby/setup-ruby@v1
-        if: |
-          matrix.runs-on != 'windows-latest'
-        with:
-          ruby-version: "3.1"
-          bundler-cache: true
-      - uses: actions-rs/toolchain@v1
-        id: rust-toolchain
-        with:
-          override: true
-          toolchain: stable
-      - name: Cache Cargo
-        uses: actions/cache@v2
-        with:
-          path: ~/.cargo
-          key: cargo-test-${{ matrix.runs-on }}-${{ steps.rust-toolchain.outputs.rustc_hash }}-${{ hashFiles('Cargo.lock') }}
-      - name: Install cargo-c
-        uses: actions-rs/cargo@v1
-        with:
-          command: install
-          args: cargo-c
       - name: Prepare on Ubuntu
         if: |
           matrix.runs-on == 'ubuntu-latest'
         run: |
-          pip install meson
+          sudo apt update
+          sudo apt install -y -V \
+            ca-certificates \
+            lsb-release \
+            wget
+          wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
+          sudo apt install -y -V \
+            ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
           sudo apt update
           sudo apt install -y -V \
             gtk-doc-tools \
+            libarrow-glib-dev \
             libgirepository1.0-dev \
             ninja-build \
             valac
-          echo "LD_LIBRARY_PATH=${PWD}/build:${LD_LIBRARY_PATH}" >> ${GITHUB_ENV}
+          pip install meson
+          echo "LD_LIBRARY_PATH=${PWD}/build:${HOME}/local/lib:${LD_LIBRARY_PATH}" >> ${GITHUB_ENV}
       - name: Prepare on macOS
         if: |
           matrix.runs-on == 'macos-latest'
         run: |
           brew update --preinstall
           brew bundle --file=Brewfile
+          echo "DYLD_LIBRARY_PATH=${PWD}/build:${HOME}/local/lib:${DYLD_LIBRARY_PATH}" >> ${GITHUB_ENV}
+          echo "XML_CATALOG_FILES=$(brew --prefix)/etc/xml/catalog" >> ${GITHUB_ENV}
       # - name: Prepare on Windows
       #   if: |
       #     matrix.runs-on == 'windows-latest'
       #   run: |
       #     pip install meson
+      - uses: ruby/setup-ruby@v1
+        if: |
+          matrix.runs-on != 'windows-latest'
+        with:
+          ruby-version: "3.1"
+          bundler-cache: true
+      - uses: actions-rs/toolchain@v1
+        id: rust-toolchain
+        with:
+          override: true
+          toolchain: stable
+      - name: Cache Cargo
+        uses: actions/cache@v2
+        with:
+          path: ~/.cargo
+          key: cargo-test-${{ matrix.runs-on }}-${{ steps.rust-toolchain.outputs.rustc_hash }}-${{ hashFiles('**/Cargo.toml') }}
+      - name: Install cargo-c
+        uses: actions-rs/cargo@v1
+        with:
+          command: install
+          args: cargo-c
       - name: Setup on non-Windows
-        # Enable this once https://github.com/mesonbuild/meson/pull/10476
-        # is fixed.
-        # if: |
-        #   matrix.runs-on != 'windows-latest'
         if: |
-          matrix.runs-on == 'ubuntu-latest'
+          matrix.runs-on != 'windows-latest'
+        # Enable -Dvapi=true and -Dwerror=true after Apache Arrow GLib
+        # 9.0.0 is released.
         run: |
-          meson setup -Ddoc=true -Dvala=true -Dwerror=true build .
+          meson setup \
+            --prefix=${HOME}/local \
+            -Ddoc=true \
+            -Dvapi=false \
+            -Dwerror=false \
+            build \
+            .
       - name: Build on non-Windows
-        # Enable this once https://github.com/mesonbuild/meson/pull/10476
-        # is fixed.
-        # if: |
-        #   matrix.runs-on != 'windows-latest'
         if: |
-          matrix.runs-on == 'ubuntu-latest'
+          matrix.runs-on != 'windows-latest'
         run: |
           ninja -C build
       - name: Build on Windows
-        # Enable this once https://github.com/mesonbuild/meson/pull/10476
-        # is fixed.
-        # if: |
-        #   matrix.runs-on == 'windows-latest'
         if: |
-          matrix.runs-on == 'windows-latest' || matrix.runs-on == 'macos-latest'
+          matrix.runs-on == 'windows-latest'
         uses: actions-rs/cargo@v1
         with:
           command: cbuild
       - name: Test
-        # Enable this once https://github.com/mesonbuild/meson/pull/10476
-        # is fixed.
-        # if: |
-        #   matrix.runs-on != 'windows-latest'
         if: |
-          matrix.runs-on == 'ubuntu-latest'
+          matrix.runs-on != 'windows-latest'
         run: |
           cd build
           bundle exec ../test/run.sh
-      - name: Run C example
-        # Enable this once https://github.com/mesonbuild/meson/pull/10476
-        # is fixed.
-        # if: |
-        #   matrix.runs-on != 'windows-latest'
+      - name: Install
         if: |
-          matrix.runs-on == 'ubuntu-latest'
+          matrix.runs-on != 'windows-latest'
         run: |
-          build/examples/sql-c
-      - name: Run Vala example
-        # Enable this once https://github.com/mesonbuild/meson/pull/10476
-        # is fixed.
-        # if: |
-        #   matrix.runs-on != 'windows-latest'
+          ninja -C build install
+      - name: Run C example
         if: |
-          matrix.runs-on == 'ubuntu-latest'
+          matrix.runs-on != 'windows-latest'
         run: |
-          build/examples/sql-vala
+          build/examples/sql-c
+      # Enable this after Apache Arrow GLib 9.0.0 is released.
+      # - name: Run Vala example
+      #   if: |
+      #     matrix.runs-on != 'windows-latest'
+      #   run: |
+      #     build/examples/sql-vala
       - name: Run Python example
-        # Enable this once https://github.com/mesonbuild/meson/pull/10476
-        # is fixed.
-        # if: |
-        #   matrix.runs-on != 'windows-latest'
         if: |
-          matrix.runs-on == 'ubuntu-latest'
+          matrix.runs-on != 'windows-latest'
         run: |
           cd build
           ../examples/sql.py
       - name: Run Ruby example
-        # Enable this once https://github.com/mesonbuild/meson/pull/10476
-        # is fixed.
-        # if: |
-        #   matrix.runs-on != 'windows-latest'
         if: |
-          matrix.runs-on == 'ubuntu-latest'
+          matrix.runs-on != 'windows-latest'
         run: |
           cd build
           ../examples/sql.rb
diff --git a/Brewfile b/Brewfile
@@ -17,6 +17,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
+brew "apache-arrow-glib"
 brew "gobject-introspection"
 brew "gtk-doc"
 brew "meson"

diff --git a/Gemfile b/Gemfile
@@ -19,5 +19,5 @@
 
 source "https://rubygems.org/"
 
-gem "gobject-introspection"
+gem "red-arrow"
 gem "test-unit"
diff --git a/build.py b/build.py
@@ -19,6 +19,7 @@
 
 import argparse
 import os
+import pathlib
 import platform
 import shutil
 import subprocess
@@ -63,6 +64,20 @@ def ln_fs(src, dest):
           f'{shared_object_base}.{version_major}')
     ln_fs(f'{shared_object_base}.{version_major}',
           f'{shared_object_base}')
+elif platform.system() == 'Darwin':
+    version_major = args.version.split('.')[0]
+    suffix = pathlib.PurePath(shared_object_base).suffix
+    shared_object_base = pathlib.PurePath(shared_object_base).with_suffix('')
+    shutil.copy2(args.shared_object,
+                 f'{shared_object_base}.{version_major}{suffix}')
+    def ln_fs(src, dest):
+        try:
+            os.remove(dest)
+        except FileNotFoundError:
+            pass
+        os.symlink(src, dest)
+    ln_fs(f'{shared_object_base}.{version_major}{suffix}',
+          f'{shared_object_base}{suffix}')
 else:
     shutil.copy2(args.shared_object, '.')
 

diff --git a/datafusion-glib/data-frame.c b/datafusion-glib/data-frame.c
@@ -116,6 +116,54 @@ gdf_data_frame_show(GDFDataFrame *data_frame, GError **error)
   }
 }
 
+/**
+ * gdf_data_frame_to_table:
+ * @data_frame: A #GDFDataFrame.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (transfer full): A #GArrowTable of this data frame, %NULL on error.
+ *
+ * Since: 8.0.0
+ */
+GArrowTable *
+gdf_data_frame_to_table(GDFDataFrame *data_frame, GError **error)
+{
+  GDFDataFramePrivate *priv = gdf_data_frame_get_instance_private(data_frame);
+  DFArrowSchema *c_abi_schema;
+  DFArrowArray **c_abi_record_batches;
+  DFError *df_error = NULL;
+  gint64 n = df_data_frame_export(priv->data_frame,
+                                  &c_abi_schema,
+                                  &c_abi_record_batches,
+                                  &df_error);
+  if (n < 0) {
+    g_set_error(error,
+                GDF_ERROR,
+                df_error_get_code(df_error),
+                "[data-frame][to-table] %s",
+                df_error_get_message(df_error));
+    df_error_free(df_error);
+    return NULL;
+  }
+
+  GArrowSchema *schema = garrow_schema_import(c_abi_schema, error);
+  GArrowRecordBatch **record_batches = g_newa(GArrowRecordBatch *, n);
+  gint64 i;
+  for (i = 0; i < n; i++) {
+    record_batches[i] = garrow_record_batch_import(c_abi_record_batches[i],
+                                                   schema,
+                                                   error);
+  }
+  free(c_abi_record_batches);
+  GArrowTable *table =
+    garrow_table_new_record_batches(schema, record_batches, n, error);
+  for (i = 0; i < n; i++) {
+    g_object_unref(record_batches[i]);
+  }
+  g_object_unref(schema);
+  return table;
+}
+
 GDFDataFrame *
 gdf_data_frame_new_raw(DFDataFrame *data_frame)
 {

diff --git a/datafusion-glib/data-frame.h b/datafusion-glib/data-frame.h
@@ -19,6 +19,7 @@
 
 #pragma once
 
+#include <arrow-glib/arrow-glib.h>
 #include <datafusion-glib/error.h>
 
 G_BEGIN_DECLS
@@ -37,5 +38,9 @@ struct _GDFDataFrameClass
 GDF_AVAILABLE_IN_8_0
 gboolean
 gdf_data_frame_show(GDFDataFrame *data_frame, GError **error);
+GDF_AVAILABLE_IN_8_0
+GArrowTable *
+gdf_data_frame_to_table(GDFDataFrame *data_frame, GError **error);
+
 
 G_END_DECLS
diff --git a/datafusion-glib/meson.build b/datafusion-glib/meson.build
@@ -59,9 +59,11 @@ enums_header = enums[1]
 install_headers(headers + raw_headers, subdir: 'datafusion-glib')
 
 
+gobject = dependency('gobject-2.0')
 dependencies = [
   datafusion,
-  dependency('gobject-2.0'),
+  arrow_glib,
+  gobject,
 ]
 libdatafusion_glib = library('datafusion-glib',
                              dependencies: dependencies,
@@ -79,19 +81,21 @@ pkgconfig.generate(libdatafusion_glib,
                    description: 'GLib API for DataFusion',
                    filebase: 'datafusion-glib',
                    name: 'DataFusion GLib',
-                   requires: ['gobject-2.0', 'datafusion'],
+                   requires: ['datafusion', 'arrow-glib', 'gobject-2.0'],
                    version: version)
 
 if have_gi
   datafusion_glib_gir = \
     gnome.generate_gir(libdatafusion_glib,
+                       dependencies: dependencies,
                        export_packages: 'datafusion-glib',
                        extra_args: [
                          '--warn-all',
                        ],
                        header: 'datafusion-glib/datafusion-glib.h',
                        identifier_prefix: 'GDF',
                        includes: [
+                         'Arrow-1.0',
                          'GObject-2.0',
                        ],
                        install: true,
@@ -103,8 +107,14 @@ if have_gi
   if generate_vapi
     datafusion_glib_vapi = \
       gnome.generate_vapi('datafusion-glib',
+                          gir_dirs: [
+                            arrow_glib.get_variable('girdir'),
+                          ],
                           install: true,
-                          packages: ['gobject-2.0'],
-                          sources: [datafusion_glib_gir[0]])
+                          packages: ['arrow-glib', 'gobject-2.0'],
+                          sources: [datafusion_glib_gir[0]],
+                          vapi_dirs: [
+                            arrow_glib.get_variable('vapidir'),
+                          ])
   endif
 endif