PHOENIX-7487: Make table level max lookback accept lookback value in seconds instead of milli-seconds to be consistent with TTL

phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTableImpl.java

@@ -275,7 +275,7 @@ public class PTableImpl implements PTable {
     private final BitSet viewModifiedPropSet;
     private final Long lastDDLTimestamp;
     private final boolean isChangeDetectionEnabled;
-    private Map<String, String> propertyValues;
+    private Map<String, String> propertyValues;  // Map of property name to value for constructing DDL dynamically
     private String schemaVersion;
     private String externalSchemaId;
     private String streamingTopicName;
@@ -781,7 +781,9 @@ public Builder setIndexWhere(String indexWhere) {
 
         public Builder setMaxLookbackAge(Long maxLookbackAge) {
             if (maxLookbackAge != null) {
-                propertyValues.put(MAX_LOOKBACK_AGE, String.valueOf(maxLookbackAge));
+                // The table level max lookback age read from SYSCAT is in milli-seconds but in DDL it should
+                // be specified in seconds.
+                propertyValues.put(MAX_LOOKBACK_AGE, String.valueOf(maxLookbackAge / 1000));
             }
             this.maxLookbackAge = maxLookbackAge;
             return this;

phoenix-core-client/src/main/java/org/apache/phoenix/schema/TableProperty.java

@@ -334,17 +334,19 @@ public Object getValue(Object value) {
     MAX_LOOKBACK_AGE(PhoenixDatabaseMetaData.MAX_LOOKBACK_AGE, true, false, false) {
         @Override
         public Object getValue(Object value) {
+            // User provides table level max lookback age in seconds in DDL and
+            // we convert to milli-seconds before storing in SYSCAT
             if (value == null) {
                 return null;
             }
             else if (value instanceof Integer) {
-                return Long.valueOf((Integer) value);
+                return Long.valueOf((Integer) value) * 1000;
             }
             else if (value instanceof Long) {
-                return value;
+                return (Long) value * 1000;
             }
             else {
-                throw new IllegalArgumentException("Table level MAX_LOOKBACK_AGE should be a " + PLong.INSTANCE.getSqlTypeName() + " value in milli-seconds");
+                throw new IllegalArgumentException("Table level MAX_LOOKBACK_AGE should be a " + PLong.INSTANCE.getSqlTypeName() + " value in seconds");
             }
         }
 

phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java

@@ -112,7 +112,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT {
     private final boolean columnEncoded;
     private static final Logger LOGGER = LoggerFactory.getLogger(AlterTableIT.class);
 
-    private final long oneDayInMillis = Duration.ofDays(1).toMillis();
+    private final long oneDayInSeconds = Duration.ofDays(1).getSeconds();
 
     public AlterTableIT(boolean columnEncoded) {
         this.columnEncoded = columnEncoded;
@@ -1805,16 +1805,16 @@ public void testChangeTableLevelMaxLookbackAge() throws Exception {
         String schemaName = generateUniqueName();
         String dataTableName = generateUniqueName();
         String fullTableName = SchemaUtil.getTableName(schemaName, dataTableName);
-        Long maxLookbackAge = oneDayInMillis;
+        Long maxLookbackAge = oneDayInSeconds;
         try (Connection conn = DriverManager.getConnection(getUrl())) {
             String ddl = "CREATE TABLE  " + fullTableName +
                     "  (a_string varchar not null PRIMARY KEY, col1 integer) MAX_LOOKBACK_AGE=" + maxLookbackAge;
             conn.createStatement().execute(ddl);
             assertMaxLookbackAge(fullTableName, maxLookbackAge);
-            maxLookbackAge = 3L * oneDayInMillis;
+            maxLookbackAge = 3L * oneDayInSeconds;
             alterTableLevelMaxLookbackAge(fullTableName, maxLookbackAge.toString());
             assertMaxLookbackAge(fullTableName, maxLookbackAge);
-            maxLookbackAge = 2L * oneDayInMillis;
+            maxLookbackAge = 2L * oneDayInSeconds;
             alterTableLevelMaxLookbackAge(fullTableName, maxLookbackAge.toString());
             assertMaxLookbackAge(fullTableName, maxLookbackAge);
             maxLookbackAge = 0L;
@@ -1830,7 +1830,7 @@ public void testChangeTableLevelMaxLookbackAgeToInvalid() throws Exception {
         String fullTableName = SchemaUtil.getTableName(schemaName, dataTableName);
         try(Connection conn = DriverManager.getConnection(getUrl())) {
             String ddl = "CREATE TABLE  " + fullTableName +
-                    "  (a_string varchar not null PRIMARY KEY, col1 integer) MAX_LOOKBACK_AGE=" + oneDayInMillis;
+                    "  (a_string varchar not null PRIMARY KEY, col1 integer) MAX_LOOKBACK_AGE=" + oneDayInSeconds;
             conn.createStatement().execute(ddl);
         }
         assertThrows(IllegalArgumentException.class, () -> alterTableLevelMaxLookbackAge(fullTableName, "2309.3"));
@@ -1842,7 +1842,7 @@ public void testMaxLookbackAgeOfChildViewsAndIndexesWithAlterTable() throws Exce
         String schemaName = generateUniqueName();
         String dataTableName = generateUniqueName();
         String fullDataTableName = SchemaUtil.getTableName(schemaName, dataTableName);
-        Long maxLookbackAge = oneDayInMillis;
+        Long maxLookbackAge = oneDayInSeconds;
         try(Connection conn = DriverManager.getConnection(getUrl());
             Statement stmt = conn.createStatement()) {
             String ddl = "CREATE TABLE  " + fullDataTableName +
@@ -1874,7 +1874,7 @@ public void testMaxLookbackAgeOfChildViewsAndIndexesWithAlterTable() throws Exce
             ddl = "CREATE INDEX " + grandChildViewIndexName + " ON " + fullGrandChildViewName + " (COL2)";
             stmt.execute(ddl);
             assertMaxLookbackAge(fullGrandChildViewIndexName, maxLookbackAge);
-            maxLookbackAge = 2 * oneDayInMillis;
+            maxLookbackAge = 2 * oneDayInSeconds;
             alterTableLevelMaxLookbackAge(fullDataTableName, maxLookbackAge.toString());
             assertMaxLookbackAge(fullDataTableName, maxLookbackAge);
             assertMaxLookbackAge(fullIndexName, maxLookbackAge);
@@ -1888,7 +1888,7 @@ public void testMaxLookbackAgeOfChildViewsAndIndexesWithAlterTable() throws Exce
     private void assertMaxLookbackAge(String fullTableName, Long expectedMaxLookbackAge) throws Exception {
         try (Connection conn = DriverManager.getConnection(getUrl())) {
             PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class);
-            assertEquals(expectedMaxLookbackAge, pconn.getTableNoCache(fullTableName).getMaxLookbackAge());
+            assertEquals((Long) (expectedMaxLookbackAge * 1000), pconn.getTableNoCache(fullTableName).getMaxLookbackAge());
         }
     }
 

phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java

@@ -1850,18 +1850,18 @@ public void testCreateTableWithTableLevelMaxLookbackAge() throws Exception {
         String schemaName = generateUniqueName();
         String dataTableName = generateUniqueName();
         String fullTableName = SchemaUtil.getTableName(schemaName, dataTableName);
-        Long maxLookbackAge = 259200L;
+        Long maxLookbackAge = 2592L;
         createTableWithTableLevelMaxLookbackAge(fullTableName, maxLookbackAge.toString());
-        assertEquals(maxLookbackAge, queryTableLevelMaxLookbackAge(fullTableName));
+        assertEquals((Long) (maxLookbackAge * 1000), queryTableLevelMaxLookbackAge(fullTableName));
         schemaName = generateUniqueName();
         dataTableName = generateUniqueName();
         fullTableName = SchemaUtil.getTableName(schemaName, dataTableName);
-        maxLookbackAge = 25920000000L;
+        maxLookbackAge = 25920000L;
         createTableWithTableLevelMaxLookbackAge(fullTableName, maxLookbackAge.toString());
-        assertEquals(maxLookbackAge, queryTableLevelMaxLookbackAge(fullTableName));
+        assertEquals((Long) (maxLookbackAge * 1000), queryTableLevelMaxLookbackAge(fullTableName));
         String indexTableName = generateUniqueName();
         createIndexOnTableWithMaxLookbackAge(indexTableName, fullTableName);
-        assertEquals(maxLookbackAge, queryTableLevelMaxLookbackAge(SchemaUtil.getTableName(schemaName, indexTableName)));
+        assertEquals((Long) (maxLookbackAge * 1000), queryTableLevelMaxLookbackAge(SchemaUtil.getTableName(schemaName, indexTableName)));
     }
 
     @Test
@@ -1878,7 +1878,7 @@ public void testCreateTableWithTableLevelMaxLookbackAgeAsNull() throws Exception
 
     @Test
     public void testCreateTableWithInvalidTableLevelMaxLookbackAge() {
-        String errMsg = "Table level MAX_LOOKBACK_AGE should be a BIGINT value in milli-seconds";
+        String errMsg = "Table level MAX_LOOKBACK_AGE should be a BIGINT value in seconds";
         IllegalArgumentException err = assertThrows(IllegalArgumentException.class,
                 () -> createTableWithTableLevelMaxLookbackAge(
                         SchemaUtil.getTableName(generateUniqueName(), generateUniqueName()), "2.3"));

phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexRepairRegionScannerIT.java

@@ -833,7 +833,7 @@ public void testInvalidIndexRowWithTableLevelMaxLookback() throws Exception {
             return;
         }
         final int NROWS = 4;
-        final int maxLookbackAge = 12000;
+        final int maxLookbackAge = 12;
         if ((optionBuilder.length() > 0 && optionBuilder.toString().trim().startsWith("SPLIT ON"))
                 || optionBuilder.length() == 0) {
             optionBuilder.insert(0, "MAX_LOOKBACK_AGE=" + maxLookbackAge + " ");
@@ -885,7 +885,7 @@ public void testInvalidIndexRowWithTableLevelMaxLookback() throws Exception {
             conn.createStatement().execute("UPSERT INTO " + dataTableFullName + " VALUES(2, 102, 202)");
             conn.commit();
             IndexRegionObserver.setFailPostIndexUpdatesForTesting(false);
-            customEdge.incrementValue(maxLookbackAge + 1);
+            customEdge.incrementValue(maxLookbackAge * 1000 + 1);
             indexTool = IndexToolIT.runIndexTool(false, schemaName, dataTableName,
                     indexTableName, null, 0, IndexVerifyType.BEFORE, "-fi");
             mrJobCounters = IndexToolIT.getMRJobCounters(indexTool);

phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexScrutinyWithMaxLookbackIT.java

@@ -230,7 +230,7 @@ private void setupTables() throws SQLException {
         String dataTableDDL = "CREATE TABLE %s (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR, "
             + "ZIP INTEGER) COLUMN_ENCODED_BYTES=0, VERSIONS=1";
         if (hasTableLevelMaxLookback) {
-            dataTableDDL += ", MAX_LOOKBACK_AGE=" + TABLE_LEVEL_MAX_LOOKBACK * 1000;
+            dataTableDDL += ", MAX_LOOKBACK_AGE=" + TABLE_LEVEL_MAX_LOOKBACK;
         }
         String indexTableDDL = "CREATE INDEX %s ON %s (NAME) INCLUDE (ZIP)";
         testClock = new ManualEnvironmentEdge();

phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForNonTxGlobalIndexIT.java

@@ -1490,7 +1490,7 @@ public void testMissingIndexRowsBeyondTableLevelMaxLookback() throws Exception {
         String dataTableName = generateUniqueName();
         String fullDataTableName = SchemaUtil.getTableName(schemaName, dataTableName);
         String indexTableName = generateUniqueName();
-        int maxLookbackAge = 12000;
+        int maxLookbackAge = 12;
         int delta = 5;
         try (Connection conn = DriverManager.getConnection(getUrl())) {
             conn.createStatement().execute("CREATE TABLE " + fullDataTableName
@@ -1525,7 +1525,7 @@ public void testMissingIndexRowsBeyondTableLevelMaxLookback() throws Exception {
                     mrJobCounters.findCounter(BEFORE_REBUILD_MISSING_INDEX_ROW_COUNT.name()).getValue());
             assertEquals(0, mrJobCounters.findCounter(
                     BEFORE_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT.name()).getValue());
-            customEdge.incrementValue(maxLookbackAge + 1);
+            customEdge.incrementValue(maxLookbackAge * 1000 + 1);
             indexTool = IndexToolIT.runIndexTool(useSnapshot, schemaName, dataTableName, indexTableName, null,
                     0, IndexTool.IndexVerifyType.BEFORE);
             mrJobCounters = IndexToolIT.getMRJobCounters(indexTool);

phoenix-core/src/it/java/org/apache/phoenix/end2end/MaxLookbackExtendedIT.java

@@ -130,7 +130,7 @@ public void testKeepDeletedCellsWithMaxLookbackAge() throws Exception {
         optionBuilder.append(", VERSIONS=" + versions);
         optionBuilder.append(", KEEP_DELETED_CELLS=TRUE");
         if(hasTableLevelMaxLookback) {
-            optionBuilder.append(", MAX_LOOKBACK_AGE=" + TABLE_LEVEL_MAX_LOOKBACK_AGE * 1000);
+            optionBuilder.append(", MAX_LOOKBACK_AGE=" + TABLE_LEVEL_MAX_LOOKBACK_AGE);
         }
         tableDDLOptions = optionBuilder.toString();
         try (Connection conn = DriverManager.getConnection(getUrl())) {
@@ -218,7 +218,7 @@ public void testKeepDeletedCellsWithMaxLookbackAge() throws Exception {
     @Test
     public void testTooLowSCNWithMaxLookbackAge() throws Exception {
         if(hasTableLevelMaxLookback) {
-            optionBuilder.append(", MAX_LOOKBACK_AGE=" + TABLE_LEVEL_MAX_LOOKBACK_AGE * 1000);
+            optionBuilder.append(", MAX_LOOKBACK_AGE=" + TABLE_LEVEL_MAX_LOOKBACK_AGE);
             tableDDLOptions = optionBuilder.toString();
         }
         String dataTableName = generateUniqueName();
@@ -252,7 +252,7 @@ public void testTooLowSCNWithMaxLookbackAge() throws Exception {
     @Test(timeout=120000L)
     public void testRecentlyDeletedRowsNotCompactedAway() throws Exception {
         if(hasTableLevelMaxLookback) {
-            optionBuilder.append(", MAX_LOOKBACK_AGE=" + TABLE_LEVEL_MAX_LOOKBACK_AGE * 1000);
+            optionBuilder.append(", MAX_LOOKBACK_AGE=" + TABLE_LEVEL_MAX_LOOKBACK_AGE);
             tableDDLOptions = optionBuilder.toString();
         }
         try (Connection conn = DriverManager.getConnection(getUrl())) {
@@ -477,7 +477,7 @@ public void testRecentMaxVersionsNotCompactedAway() throws Exception {
         int versions = 2;
         optionBuilder.append(", VERSIONS=" + versions);
         if(hasTableLevelMaxLookback) {
-            optionBuilder.append(", MAX_LOOKBACK_AGE=" + TABLE_LEVEL_MAX_LOOKBACK_AGE * 1000);
+            optionBuilder.append(", MAX_LOOKBACK_AGE=" + TABLE_LEVEL_MAX_LOOKBACK_AGE);
         }
         tableDDLOptions = optionBuilder.toString();
         String firstValue = "abc";
@@ -562,7 +562,7 @@ public void testRecentMaxVersionsNotCompactedAway() throws Exception {
     @Test(timeout=60000)
     public void testOverrideMaxLookbackForCompaction() throws Exception {
         if(hasTableLevelMaxLookback) {
-            optionBuilder.append(", MAX_LOOKBACK_AGE=" + TABLE_LEVEL_MAX_LOOKBACK_AGE * 1000);
+            optionBuilder.append(", MAX_LOOKBACK_AGE=" + TABLE_LEVEL_MAX_LOOKBACK_AGE);
             tableDDLOptions = optionBuilder.toString();
         }
         try (Connection conn = DriverManager.getConnection(getUrl())) {
@@ -621,7 +621,7 @@ public void testOverrideMaxLookbackForCompaction() throws Exception {
     @Test(timeout=60000)
     public void testRetainingLastRowVersion() throws Exception {
         if(hasTableLevelMaxLookback) {
-            optionBuilder.append(", MAX_LOOKBACK_AGE=" + TABLE_LEVEL_MAX_LOOKBACK_AGE * 1000);
+            optionBuilder.append(", MAX_LOOKBACK_AGE=" + TABLE_LEVEL_MAX_LOOKBACK_AGE);
             tableDDLOptions = optionBuilder.toString();
         }
         try(Connection conn = DriverManager.getConnection(getUrl())) {

phoenix-core/src/it/java/org/apache/phoenix/end2end/MaxLookbackIT.java

@@ -126,11 +126,11 @@ public void testTooLowSCNWithMaxLookbackAge() throws Exception {
     public void testTooLowSCNWithTableLevelMaxLookback() throws Exception {
         String dataTableName1 = generateUniqueName();
         StringBuilder optionBuilderForDataTable1 = new StringBuilder(optionBuilder);
-        optionBuilderForDataTable1.append("MAX_LOOKBACK_AGE="+((TABLE_LEVEL_MAX_LOOKBACK_AGE + 3) * 1000));
+        optionBuilderForDataTable1.append("MAX_LOOKBACK_AGE=" + (TABLE_LEVEL_MAX_LOOKBACK_AGE + 3));
         tableDDLOptions = optionBuilderForDataTable1.toString();
         createTable(dataTableName1);
         StringBuilder optionBuilderForDataTable2 = new StringBuilder(optionBuilder);
-        optionBuilderForDataTable2.append("MAX_LOOKBACK_AGE="+(TABLE_LEVEL_MAX_LOOKBACK_AGE * 1000));
+        optionBuilderForDataTable2.append("MAX_LOOKBACK_AGE=" + TABLE_LEVEL_MAX_LOOKBACK_AGE);
         String dataTableName2 = generateUniqueName();
         tableDDLOptions = optionBuilderForDataTable2.toString();
         createTable(dataTableName2);
@@ -235,7 +235,7 @@ public void testRecentlyDeletedRowsNotCompactedAway() throws Exception {
     public void testRecentlyDeletedRowsNotCompactedAwayWithTableLevelMaxLookback() throws Exception {
         try(Connection conn = DriverManager.getConnection(getUrl())) {
             String dataTableName = generateUniqueName();
-            optionBuilder.append("MAX_LOOKBACK_AGE="+(TABLE_LEVEL_MAX_LOOKBACK_AGE * 1000));
+            optionBuilder.append("MAX_LOOKBACK_AGE=" + TABLE_LEVEL_MAX_LOOKBACK_AGE);
             tableDDLOptions = optionBuilder.toString();
             String indexName = generateUniqueName();
             createTable(dataTableName);
@@ -377,7 +377,7 @@ public void testTTLAndTableLevelMaxLookbackWithMaxLookbackMoreThanTTL() throws E
         ttl = 8;
         long maxLookbackAge = 12;
         optionBuilder.append("TTL=").append(ttl).append(", MAX_LOOKBACK_AGE=").
-                append(maxLookbackAge * 1000);
+                append(maxLookbackAge);
         tableDDLOptions = optionBuilder.toString();
         Configuration conf = getUtility().getConfiguration();
         //disable automatic memstore flushes
@@ -455,7 +455,7 @@ public void testExpiredRowsAreNotFlushedWithTableLevelMaxLookback() throws Excep
         ttl = 8;
         long maxLookbackAge = 12;
         optionBuilder.append("TTL=").append(ttl).append(", MAX_LOOKBACK_AGE=").
-                append(maxLookbackAge * 1000);
+                append(maxLookbackAge);
         tableDDLOptions = optionBuilder.toString();
         int delta = 1;
         Configuration conf = getUtility().getConfiguration();

phoenix-core/src/it/java/org/apache/phoenix/end2end/TableTTLIT.java

@@ -111,7 +111,7 @@ public void beforeTest(){
             optionBuilder.append(", COLUMN_ENCODED_BYTES=0");
         }
         if (tableLevelMaxLooback != null) {
-            optionBuilder.append(", MAX_LOOKBACK_AGE=").append(tableLevelMaxLooback * 1000);
+            optionBuilder.append(", MAX_LOOKBACK_AGE=").append(tableLevelMaxLooback);
         }
         this.tableDDLOptions = optionBuilder.toString();
         injectEdge = new ManualEnvironmentEdge();