Adding Statistics (#1128)

* Adding Statistics

* Adding statistics

gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsGlobalStorageStatistics.java

@@ -16,9 +16,6 @@
 
 package com.google.cloud.hadoop.fs.gcs;
 
-import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.DIRECTORIES_CREATED;
-import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.FILES_CREATED;
-import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.FILES_DELETED;
 import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.EXCEPTION_COUNT;
 import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_CLIENT_RATE_LIMIT_COUNT;
 import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_CLIENT_SIDE_ERROR_COUNT;
@@ -32,6 +29,7 @@
 import com.google.api.client.http.HttpResponse;
 import com.google.api.client.http.HttpResponseException;
 import com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics;
+import com.google.cloud.hadoop.util.GoogleCloudStorageEventBus.StatisticsType;
 import com.google.cloud.hadoop.util.ITraceFactory;
 import com.google.cloud.hadoop.util.ITraceOperation;
 import com.google.common.base.Stopwatch;
@@ -61,7 +59,7 @@ public class GhfsGlobalStorageStatistics extends StorageStatistics {
   /** {@value} The key that stores all the registered metrics */
   public static final String NAME = "GhfsStorageStatistics";
 
-  public static final int LATENCY_LOGGING_THRESHOLD_MS = 150;
+  public static final int LATENCY_LOGGING_THRESHOLD_MS = 300;
 
   // Instance to be used if it encounters any error while registering to Global Statistics.
   // Error can happen for e.g. when different class loaders are used.
@@ -289,6 +287,22 @@ private void subscriberOnException(IOException exception) {
     incrementGcsExceptionCount();
   }
 
+  /**
+   * Updating the corresponding statistics
+   *
+   * @param strType
+   */
+  @Subscribe
+  private void subscriberOnStatisticsType(StatisticsType strType) {
+    if (strType == StatisticsType.DIRECTORIES_DELETED) {
+      incrementDirectoriesDeleted();
+    }
+  }
+
+  private void incrementDirectoriesDeleted() {
+    increment(GhfsStatistic.DIRECTORIES_DELETED);
+  }
+
   private void incrementGcsExceptionCount() {
     increment(EXCEPTION_COUNT);
   }
@@ -337,18 +351,6 @@ void streamWriteBytes(int bytesWritten) {
     incrementCounter(GhfsStatistic.STREAM_WRITE_BYTES, bytesWritten);
   }
 
-  void filesCreated() {
-    increment(FILES_CREATED);
-  }
-
-  public void fileDeleted(int count) {
-    incrementCounter(FILES_DELETED, count);
-  }
-
-  public void directoryCreated() {
-    incrementCounter(DIRECTORIES_CREATED, 1);
-  }
-
   private class LongIterator implements Iterator<LongStatistic> {
     private Iterator<String> iterator = getMetricNames();
 

gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsInstrumentation.java

@@ -17,9 +17,6 @@
 package com.google.cloud.hadoop.fs.gcs;
 
 import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.DELEGATION_TOKENS_ISSUED;
-import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.DIRECTORIES_CREATED;
-import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.FILES_CREATED;
-import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.FILES_DELETED;
 import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.INVOCATION_HFLUSH;
 import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.INVOCATION_HSYNC;
 import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.STREAM_WRITE_BYTES;
@@ -200,19 +197,6 @@ public void incrementCounter(GhfsStatistic op, long count) {
     instanceIOStatistics.incrementCounter(name, count);
   }
 
-  /**
-   * Increments a mutable counter and the matching instance IOStatistics counter for metrics in
-   * GoogleCloudStorageStatusStatistics.
-   *
-   * @param op operation
-   */
-  private void incrementCounter(GoogleCloudStorageStatistics op) {
-
-    String name = op.getSymbol();
-    incrementMutableCounter(name, 1);
-    instanceIOStatistics.incrementCounter(name, 1);
-  }
-
   /**
    * Get the metrics system.
    *
@@ -416,25 +400,6 @@ public DurationTracker trackDuration(String key, long count) {
   @Override
   public void getMetrics(MetricsCollector metricsCollector, boolean b) {}
 
-  /** Indicate that GCS created a file. */
-  public void fileCreated() {
-    incrementCounter(FILES_CREATED, 1);
-  }
-
-  /** Indicate that GCS created a directory. */
-  public void directoryCreated() {
-    incrementCounter(DIRECTORIES_CREATED, 1);
-  }
-
-  /**
-   * Indicate that GCS deleted one or more files.
-   *
-   * @param count number of files.
-   */
-  public void fileDeleted(int count) {
-    incrementCounter(FILES_DELETED, count);
-  }
-
   /**
    * Create a stream input statistics instance.
    *

gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsStatistic.java

@@ -55,6 +55,11 @@ public enum GhfsStatistic {
       "Total number of directories created through the object store.",
       TYPE_COUNTER),
 
+  DIRECTORIES_DELETED(
+      "directories_deleted",
+      "Total number of directories deleted through the object store.",
+      TYPE_COUNTER),
+
   FILES_CREATED(
       "files_created", "Total number of files created through the object store.", TYPE_COUNTER),
   FILES_DELETED(
@@ -70,6 +75,9 @@ public enum GhfsStatistic {
       StoreStatisticNames.OP_GET_FILE_STATUS, "Calls of getFileStatus()", TYPE_COUNTER),
   INVOCATION_GET_FILE_CHECKSUM(
       StoreStatisticNames.OP_GET_FILE_CHECKSUM, "Calls of getFileChecksum()", TYPE_COUNTER),
+
+  INVOCATION_LIST_STATUS_RESULT_SIZE(
+      "op_get_list_status_result_size", "Number of files returned from list call", TYPE_COUNTER),
   INVOCATION_GLOB_STATUS(
       StoreStatisticNames.OP_GLOB_STATUS, "Calls of globStatus()", TYPE_DURATION),
   INVOCATION_HFLUSH(StoreStatisticNames.OP_HFLUSH, "Calls of hflush()", TYPE_DURATION),

gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystem.java

@@ -593,8 +593,8 @@ public FSDataOutputStream create(
                           .build(),
                       statistics),
                   statistics);
-          globalStorageStatistics.filesCreated();
-          instrumentation.fileCreated();
+
+          incrementStatistic(GhfsStatistic.FILES_CREATED);
           return response;
         });
   }
@@ -726,8 +726,8 @@ public boolean delete(Path hadoopPath, boolean recursive) throws IOException {
                   "delete(hadoopPath: %s, recursive: %b): true", hadoopPath, recursive);
             }
             response = result;
-            globalStorageStatistics.fileDeleted(1);
-            instrumentation.fileDeleted(1);
+
+            incrementStatistic(GhfsStatistic.FILES_DELETED);
           } catch (IOException e) {
             GoogleCloudStorageEventBus.postOnException();
             incrementStatistic(GhfsStatistic.FILES_DELETE_REJECTED);
@@ -765,6 +765,8 @@ public FileStatus[] listStatus(Path hadoopPath) throws IOException {
                       "listStatus(hadoopPath: %s): '%s' does not exist.", hadoopPath, gcsPath))
               .initCause(fnfe);
     }
+
+    incrementStatistic(GhfsStatistic.INVOCATION_LIST_STATUS_RESULT_SIZE, status.size());
     return status.toArray(new FileStatus[0]);
   }
 
@@ -790,8 +792,8 @@ public boolean mkdirs(Path hadoopPath, FsPermission permission) throws IOExcepti
     }
     logger.atFiner().log("mkdirs(hadoopPath: %s, permission: %s): true", hadoopPath, permission);
     boolean response = true;
-    instrumentation.directoryCreated();
-    globalStorageStatistics.directoryCreated();
+
+    incrementStatistic(GhfsStatistic.DIRECTORIES_CREATED);
     return response;
   }
 
@@ -1014,7 +1016,6 @@ public List<String> listXAttrs(Path path) throws IOException {
    */
   private void incrementStatistic(GhfsStatistic statistic) {
     incrementStatistic(statistic, 1);
-    globalStorageStatistics.incrementCounter(statistic, 1);
   }
 
   /**
@@ -1028,6 +1029,7 @@ private void incrementStatistic(GhfsStatistic statistic, long count) {
       return;
     }
     instrumentation.incrementCounter(statistic, count);
+    globalStorageStatistics.incrementCounter(statistic, count);
   }
 
   /**

gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemIntegrationTest.java

@@ -20,6 +20,7 @@
 import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.ACTION_HTTP_GET_REQUEST;
 import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.ACTION_HTTP_PATCH_REQUEST;
 import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.ACTION_HTTP_PUT_REQUEST;
+import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.DIRECTORIES_DELETED;
 import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.FILES_CREATED;
 import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.INVOCATION_COPY_FROM_LOCAL_FILE;
 import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.INVOCATION_CREATE;
@@ -32,6 +33,7 @@
 import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.INVOCATION_LIST_FILES;
 import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.INVOCATION_LIST_LOCATED_STATUS;
 import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.INVOCATION_LIST_STATUS;
+import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.INVOCATION_LIST_STATUS_RESULT_SIZE;
 import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.INVOCATION_MKDIRS;
 import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.INVOCATION_OPEN;
 import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.INVOCATION_OP_XATTR_LIST;
@@ -505,6 +507,77 @@ public void delete_IOstatistics() throws IOException {
         (GhfsGlobalStorageStatistics) stats, INVOCATION_DELETE.getSymbol(), 1);
   }
 
+  @Test
+  public void statistics_check_directories_deleted() throws IOException {
+
+    GoogleHadoopFileSystem myGhfs = createInMemoryGoogleHadoopFileSystem();
+
+    StorageStatistics GlobalStorageStats = TestUtils.getStorageStatistics();
+    Path testRoot = new Path("/directory1/");
+    myGhfs.mkdirs(testRoot);
+    FSDataOutputStream fout = myGhfs.create(new Path("/directory1/file1"));
+    fout.writeBytes("Test Content");
+    fout.close();
+
+    assertThat(myGhfs.delete(testRoot, /* recursive= */ true)).isTrue();
+    TestUtils.verifyCounter(
+        (GhfsGlobalStorageStatistics) GlobalStorageStats, DIRECTORIES_DELETED, 1);
+  }
+
+  @Test
+  public void statistics_check_get_list_status_result_size() throws IOException {
+
+    // first filesystem object
+    GoogleHadoopFileSystem myGhfs1 = createInMemoryGoogleHadoopFileSystem();
+    StorageStatistics stats = TestUtils.getStorageStatistics();
+
+    Path testRoot = new Path("/directory1/");
+
+    // first file created in ghfs1
+    myGhfs1.mkdirs(testRoot);
+    FSDataOutputStream fout = myGhfs1.create(new Path("/directory1/file1"));
+    fout.writeBytes("data");
+    fout.close();
+    myGhfs1.listStatus(testRoot);
+    assertThat(
+            (myGhfs1)
+                .getIOStatistics()
+                .counters()
+                .get(INVOCATION_LIST_STATUS_RESULT_SIZE.getSymbol()))
+        .isEqualTo(1);
+
+    TestUtils.verifyCounter(
+        (GhfsGlobalStorageStatistics) stats, INVOCATION_LIST_STATUS_RESULT_SIZE, 1);
+    assertThat(myGhfs1.delete(testRoot, /* recursive= */ true)).isTrue();
+
+    // create another FileSystem Object
+    GoogleHadoopFileSystem myGhfs2 = createInMemoryGoogleHadoopFileSystem();
+
+    // first file created in ghfs2
+    fout = myGhfs2.create(new Path("/directory1/file1"));
+    fout.writeBytes("data");
+    fout.close();
+
+    // first file created in ghfs3
+    fout = myGhfs2.create(new Path("/directory1/file2"));
+    fout.writeBytes("data");
+    fout.close();
+
+    myGhfs2.listStatus(testRoot);
+
+    assertThat(
+            (myGhfs2)
+                .getIOStatistics()
+                .counters()
+                .get(INVOCATION_LIST_STATUS_RESULT_SIZE.getSymbol()))
+        .isEqualTo(2);
+
+    TestUtils.verifyCounter(
+        (GhfsGlobalStorageStatistics) stats, INVOCATION_LIST_STATUS_RESULT_SIZE, 3);
+
+    assertThat(myGhfs2.delete(testRoot, /* recursive= */ true)).isTrue();
+  }
+
   @Test
   public void statistics_check_read_twice() throws Exception {
     GoogleHadoopFileSystem fs1 = createInMemoryGoogleHadoopFileSystem();

gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemImpl.java

@@ -342,6 +342,7 @@ public void delete(URI path, boolean recursive) throws IOException {
         GoogleCloudStorageEventBus.postOnException();
         throw new DirectoryNotEmptyException("Cannot delete a non-empty directory.");
       }
+      GoogleCloudStorageEventBus.postOnStatisticsType();
     } else {
       itemsToDelete = new ArrayList<>();
     }

util/src/main/java/com/google/cloud/hadoop/util/GoogleCloudStorageEventBus.java

@@ -26,6 +26,11 @@
 /** Event Bus class */
 public class GoogleCloudStorageEventBus {
 
+  /** Translates increment of statistics from API calls into StatisticsType */
+  public enum StatisticsType {
+    DIRECTORIES_DELETED
+  }
+
   /** Hold the instance of the event bus here */
   private static EventBus eventBus = new EventBus();
 
@@ -83,4 +88,12 @@ public static void postOnHttpRequest(HttpRequest request) {
   public static void postOnException() {
     eventBus.post(exception);
   }
+
+  /**
+   * Posting StatisticsType to invoke corresponding Subscriber method. Passing an Object as EventBus
+   * has @ElementTypesAreNonnullByDefault annotation.
+   */
+  public static void postOnStatisticsType() {
+    eventBus.post(StatisticsType.DIRECTORIES_DELETED);
+  }
 }