PHOENIX-7726 :- Handle Unknown Roles scenarios in CRR on Clients coming from servers

phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/ClusterRoleRecord.java

@@ -221,6 +221,11 @@ public boolean hasSameInfo(ClusterRoleRecord other) {
     return haGroupName.equals(other.haGroupName) && policy.equals(other.policy);
   }
 
+  /** Returns true if CRR has any url in UNKNOWN role/state. */
+  public boolean hasUnknownRole() {
+    return role1 == ClusterRole.UNKNOWN || role2 == ClusterRole.UNKNOWN;
+  }
+
   /** Returns role by url or UNKNOWN if the Url does not belong to this HA group */
   public ClusterRole getRole(String url) {
     if (url1.equals(url)) {

phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HighAvailabilityGroup.java

@@ -954,12 +954,53 @@ private ClusterRoleRecord getClusterRoleRecordFromEndpoint() throws SQLException
       Long.parseLong(properties.getProperty(PHOENIX_HA_CRR_POLLER_INTERVAL_MS_KEY, config
         .get(PHOENIX_HA_CRR_POLLER_INTERVAL_MS_KEY, PHOENIX_HA_CRR_POLLER_INTERVAL_MS_DEFAULT)));
 
-    // Get the CRR via RSEndpoint for cluster 1
     try {
-      return GetClusterRoleRecordUtil.fetchClusterRoleRecord(info.getUrl1(), info.getName(), this,
-        pollerInterval, properties);
+      // Get the CRR via RSEndpoint for cluster 1
+      ClusterRoleRecord roleRecord = GetClusterRoleRecordUtil.fetchClusterRoleRecord(info.getUrl1(),
+        info.getName(), this, pollerInterval, properties);
+      // If we have unknown role for any cluster then try getting CRR from cluster 2 endpoint and if
+      // we get unknown role from there as well then CRR with higher adminVersion wins.
+      if (roleRecord.hasUnknownRole()) {
+        ClusterRoleRecord roleRecordFromPR;
+        try {
+          roleRecordFromPR = GetClusterRoleRecordUtil.fetchClusterRoleRecord(info.getUrl2(),
+            info.getName(), this, pollerInterval, properties);
+        } catch (Exception e) {
+          // As we were able to get CRR from cluster 1 but cluster 2 threw exception then just
+          // return
+          // CRR from cluster 1 and consume this exception
+          LOG.warn("Role Record from cluster {} has Unknown Role but cluster {} threw exception, "
+            + "returning {} as CRR", info.getUrl1(), info.getUrl2(), roleRecord.toPrettyString());
+          return roleRecord;
+        }
+        if (roleRecordFromPR.hasUnknownRole()) {
+          return roleRecord.getVersion() > roleRecordFromPR.getVersion()
+            ? roleRecord
+            : roleRecordFromPR;
+        } else {
+          return roleRecordFromPR;
+        }
+      } else {
+        return roleRecord;
+      }
     } catch (Exception e) {
-      // Got exception from cluster 1 when trying to get CRR, try cluster 2
+      // If we get CRR Not Found on cluster 1, we should still try cluster 2, maybe
+      // haGroupStoreClient
+      // was not initialized somehow, but if we get any exception from cluster 2 too then we should
+      // throw CRR not found so that fallback can happen.
+      if (
+        e instanceof SQLException && ((SQLException) e).getErrorCode()
+            == SQLExceptionCode.CLUSTER_ROLE_RECORD_NOT_FOUND.getErrorCode()
+      ) {
+        try {
+          return GetClusterRoleRecordUtil.fetchClusterRoleRecord(info.getUrl2(), info.getName(),
+            this, pollerInterval, properties);
+        } catch (Exception ignoredEx) {
+          throw (SQLException) e;
+        }
+      }
+
+      // If caught exception is not CRR not found, then just try cluster 2 endpoint.
       return GetClusterRoleRecordUtil.fetchClusterRoleRecord(info.getUrl2(), info.getName(), this,
         pollerInterval, properties);
     }

phoenix-core-client/src/main/java/org/apache/phoenix/util/GetClusterRoleRecordUtil.java

@@ -142,7 +142,6 @@ private static ClusterRoleRecord getClusterRoleRecord(String url, String haGroup
   public static ClusterRoleRecord fetchClusterRoleRecord(String url, String haGroupName,
     HighAvailabilityGroup haGroup, long pollerInterval, Properties properties) throws SQLException {
     ClusterRoleRecord clusterRoleRecord = getClusterRoleRecord(url, haGroupName, true, properties);
-    // TODO: Will need to handle UNKNOWN role cases...
     if (
       clusterRoleRecord.getPolicy() == HighAvailabilityPolicy.FAILOVER
         && !clusterRoleRecord.getRole1().isActive() && !clusterRoleRecord.getRole2().isActive()

phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityGroup2IT.java

@@ -323,6 +323,33 @@ public void testNotFallbackToSingleConnection() throws Exception {
     }
   }
 
+  /**
+   * Test Fallback to Single Cluster if non Key cluster is down
+   */
+  @Test
+  public void testFallbackToSingleClusterIfNonKeyClusterIsDown() throws Exception {
+    final String tableName = RandomStringUtils.randomAlphabetic(10);
+    CLUSTERS.createTableOnClusterPair(haGroup, tableName);
+    String haGroupName2 = testName.getMethodName() + RandomStringUtils.randomAlphabetic(3);
+
+    String firstClusterUrl = CLUSTERS.getJdbcUrl1(haGroup);
+    clientProperties.setProperty(PHOENIX_HA_GROUP_ATTR, haGroupName2);
+    clientProperties.setProperty(PHOENIX_HA_FALLBACK_CLUSTER_KEY, firstClusterUrl);
+
+    // Here cluster 2 is down and fallback key is cluster 1
+    CLUSTERS.doTestWhenOneZKDown(CLUSTERS.getHBaseCluster2(), () -> {
+      try {
+        // Should get Fallback connection as fallback key is cluster 1 and cluster 2 is down
+        Connection conn = DriverManager.getConnection(jdbcHAUrl, clientProperties);
+        assertTrue(conn instanceof PhoenixConnection);
+        assertEquals(firstClusterUrl, ((PhoenixConnection) conn).getURL());
+        doTestBasicOperationsWithConnection(conn, tableName, haGroupName2);
+      } catch (SQLException e) {
+        fail("Should have failed since one HA group can not initialized. Not falling back");
+      }
+    });
+  }
+
   /**
    * Test that poller should be running when we detect a non-active role record and should stop when
    * we detect an active role record