BUG#35548572 Support OCI Object Storage Dedicated Endpoints in D&L utilities

Shell did not recognize the PAR URLs which use the dedicated endpoints:
  https://<namespace>.objectstorage.<region>.oci.customer-oci.com
It treated them as generic HTTPS URLs, and failed the dump/load
operations.

This patch adds support for this format, keeping the backwards
compatibility with the old format:
  https://objectstorage.<region>.oraclecloud.com

Change-Id: I80d53926f79cf75fdc550d85f27bda2d34d3ddd5

Author: Paweł Andruszkiewicz

modules/util/dump/dump_manifest.cc

@@ -26,6 +26,7 @@
 #include <chrono>
 #include <ctime>
 #include <stdexcept>
+#include <unordered_set>
 #include <utility>
 
 #include "mysqlshdk/include/shellcore/scoped_contexts.h"
@@ -545,7 +546,7 @@ Dump_manifest_reader::Dump_manifest_reader(
     m_reader = std::make_shared<Manifest_reader>(
         std::make_unique<mysqlshdk::storage::backend::Http_object>(
             mysqlshdk::oci::anonymize_par(config->par().full_url()), true),
-        m_config->par().object_prefix);
+        m_config->par().object_prefix());
   }
 }
 
@@ -556,35 +557,8 @@ std::string Dump_manifest_reader::join_path(const std::string &a,
 
 std::unique_ptr<mysqlshdk::storage::IFile> Dump_manifest_reader::file(
     const std::string &name, const mysqlshdk::storage::File_options &) const {
-  using mysqlshdk::oci::PAR_structure;
-  using mysqlshdk::oci::PAR_type;
-  // On read mode, the first file to be created is the handle for the manifest
-  // file, if it has not been created then we skip the validation below for
-  // additional files
-  //
-  // In READ mode, if a PAR is provided (i.e. a PAR to the progress file),
-  // then the file is "created" and it is kept on a different registry (i.e.
-  // not part of the manifest)
-  PAR_structure data;
-
-  if (parse_par(name, &data) != PAR_type::NONE) {
-    if (data.region == m_config->par().region &&
-        data.domain == m_config->par().domain &&
-        data.ns_name == m_config->par().ns_name &&
-        data.bucket == m_config->par().bucket &&
-        data.object_prefix == m_config->par().object_prefix) {
-      m_created_objects.emplace(std::move(data.object_name),
-                                File_info{data.object_path(), 0});
-
-      return std::make_unique<mysqlshdk::storage::backend::Http_object>(
-          mysqlshdk::oci::anonymize_par(name), true);
-    } else {
-      THROW_ERROR(SHERR_LOAD_MANIFEST_PAR_MISMATCH, name.c_str());
-    }
-  } else {
-    return std::make_unique<Http_manifest_object>(
-        m_reader, m_config, m_config->par().object_prefix + name);
-  }
+  return std::make_unique<Http_manifest_object>(
+      m_reader, m_config, m_config->par().object_prefix() + name);
 }
 
 std::unordered_set<File_info> Dump_manifest_reader::list_files(bool) const {
@@ -593,7 +567,7 @@ std::unordered_set<File_info> Dump_manifest_reader::list_files(bool) const {
   }
 
   // File list must exclude the PAR portion
-  const auto prefix_length = m_config->par().object_prefix.length();
+  const auto prefix_length = m_config->par().object_prefix().length();
   std::unordered_set<IDirectory::File_info> files;
 
   for (const auto &object : m_reader->list_objects()) {

modules/util/dump/dump_manifest.h

@@ -28,6 +28,7 @@
 #include <stdexcept>
 #include <string>
 #include <unordered_map>
+#include <unordered_set>
 #include <utility>
 #include <vector>
 
@@ -213,7 +214,6 @@ class Dump_manifest_reader : public mysqlshdk::storage::IDirectory {
  private:
   std::shared_ptr<Manifest_reader> m_reader;
   Dump_manifest_read_config_ptr m_config;
-  mutable std::unordered_map<std::string, File_info> m_created_objects;
 };
 
 }  // namespace dump

modules/util/load/load_dump_options.cc

@@ -26,6 +26,8 @@
 #include <mysqld_error.h>
 
 #include <algorithm>
+#include <regex>
+#include <utility>
 
 #include "modules/mod_utils.h"
 #include "modules/util/common/dump/utils.h"
@@ -440,10 +442,13 @@ void Load_dump_options::on_unpacked_options() {
 
 void Load_dump_options::on_log_options(const char *msg) const {
   std::string s = msg;
-  const auto pos = s.find("\"progressFile\":\"https://objectstorage.");
 
-  if (std::string::npos != pos) {
-    s = mysqlshdk::oci::hide_par_secret(s, pos);
+  static const std::regex k_par_progress_file(
+      R"("progressFile":"https://(?:[^\.]+\.)?objectstorage\.)");
+  std::smatch match;
+
+  if (std::regex_search(s, match, k_par_progress_file)) {
+    s = mysqlshdk::oci::hide_par_secret(s, match.position());
   }
 
   log_info("Load options: %s", s.c_str());

mysqlshdk/libs/oci/oci_par.cc

@@ -39,18 +39,16 @@ namespace {
 const std::string k_at_manifest_json = "@.manifest.json";
 
 const std::regex k_full_par_parser(
-    R"(^https:\/\/objectstorage\.([^\.]+)\.([^\/]+)\/p\/(.+)\/n\/(.+)\/b\/(.*)\/o\/((.*)\/)?(.*)$)");
+    R"(^(https:\/\/(?:[^\.]+\.)?objectstorage\.[^\/]+)\/p\/(.+)\/n\/(.+)\/b\/(.*)\/o\/((?:.*)\/)?(.*)$)");
 
 namespace par_tokens {
 
-const size_t REGION = 1;
-const size_t FULL_DOMAIN = 2;
-const size_t PARID = 3;
-const size_t NAMESPACE = 4;
-const size_t BUCKET = 5;
-const size_t PREFIX = 6;
-[[maybe_unused]] const size_t DIRNAME = 7;
-const size_t BASENAME = 8;
+const size_t ENDPOINT = 1;
+const size_t PAR_ID = 2;
+const size_t NAMESPACE = 3;
+const size_t BUCKET = 4;
+const size_t PREFIX = 5;
+const size_t BASENAME = 6;
 
 }  // namespace par_tokens
 
@@ -127,26 +125,6 @@ std::string hide_par_secret(const std::string &par, std::size_t start_at) {
   return par.substr(0, p + 3) + "<secret>" + par.substr(n);
 }
 
-std::string PAR_structure::full_url() const {
-  return par_url() + db::uri::pctencode_path(object_prefix + object_name);
-}
-
-std::string PAR_structure::par_url() const {
-  return shcore::str_format("%s/p/%s/n/%s/b/%s/o/", endpoint().c_str(),
-                            par_id.c_str(), ns_name.c_str(), bucket.c_str());
-}
-
-std::string PAR_structure::object_path() const {
-  return shcore::str_format("/p/%s/n/%s/b/%s/o/%s%s", par_id.c_str(),
-                            ns_name.c_str(), bucket.c_str(),
-                            object_prefix.c_str(), object_name.c_str());
-}
-
-std::string PAR_structure::endpoint() const {
-  return shcore::str_format("https://objectstorage.%s.%s", region.c_str(),
-                            domain.c_str());
-}
-
 PAR_type parse_par(const std::string &url, PAR_structure *data) {
   PAR_type ret_val = PAR_type::NONE;
   std::smatch results;
@@ -165,14 +143,24 @@ PAR_type parse_par(const std::string &url, PAR_structure *data) {
     }
 
     if (data) {
-      data->region = results[par_tokens::REGION];
-      data->domain = results[par_tokens::FULL_DOMAIN];
-      data->par_id = results[par_tokens::PARID];
-      data->ns_name = results[par_tokens::NAMESPACE];
-      data->bucket = results[par_tokens::BUCKET];
-      data->object_prefix =
+      data->m_endpoint = results[par_tokens::ENDPOINT];
+      const auto par_id = results[par_tokens::PAR_ID].str();
+      const auto ns_name = results[par_tokens::NAMESPACE].str();
+      const auto bucket = results[par_tokens::BUCKET].str();
+      data->m_object_prefix =
           shcore::pctdecode(results[par_tokens::PREFIX].str());
-      data->object_name = std::move(object_name);
+      data->m_object_name = std::move(object_name);
+
+      data->m_object_path = shcore::str_format(
+          "/p/%s/n/%s/b/%s/o/%s%s", par_id.c_str(), ns_name.c_str(),
+          bucket.c_str(), data->object_prefix().c_str(),
+          data->object_name().c_str());
+      data->m_par_url =
+          shcore::str_format("%s/p/%s/n/%s/b/%s/o/", data->endpoint().c_str(),
+                             par_id.c_str(), ns_name.c_str(), bucket.c_str());
+      data->m_full_url =
+          data->par_url() +
+          db::uri::pctencode_path(data->object_prefix() + data->object_name());
     }
   }
 
@@ -189,7 +177,7 @@ std::string IPAR_config::describe_self() const {
 }
 
 std::string IPAR_config::describe_url(const std::string &) const {
-  return "prefix='" + m_par.object_prefix + "'";
+  return "prefix='" + m_par.object_prefix() + "'";
 }
 
 std::unique_ptr<storage::IFile> General_par_config::file(

mysqlshdk/libs/oci/oci_par.h

@@ -61,24 +61,38 @@ struct PAR {
 enum class PAR_type { MANIFEST, PREFIX, GENERAL, NONE };
 
 struct PAR_structure {
-  std::string region;
-  std::string domain;
-  std::string par_id;
-  std::string ns_name;
-  std::string bucket;
-  std::string object_prefix;
-  std::string object_name;
+  inline PAR_type type() const noexcept { return m_type; }
+
+  inline const std::string &endpoint() const noexcept { return m_endpoint; }
+
+  inline const std::string &object_prefix() const noexcept {
+    return m_object_prefix;
+  }
+
+  inline const std::string &object_name() const noexcept {
+    return m_object_name;
+  }
 
-  std::string full_url() const;
-  std::string par_url() const;
-  std::string object_path() const;
-  std::string endpoint() const;
-  PAR_type type() const { return m_type; }
+  inline const std::string &object_path() const noexcept {
+    return m_object_path;
+  }
+
+  inline const std::string &par_url() const noexcept { return m_par_url; }
+
+  inline const std::string &full_url() const noexcept { return m_full_url; }
 
  private:
   friend PAR_type parse_par(const std::string &, PAR_structure *);
 
   PAR_type m_type = PAR_type::NONE;
+
+  std::string m_endpoint;
+  std::string m_object_prefix;
+  std::string m_object_name;
+
+  std::string m_object_path;
+  std::string m_par_url;
+  std::string m_full_url;
 };
 
 std::string to_string(PAR_access_type access_type);
@@ -112,7 +126,7 @@ class IPAR_config : public storage::Config {
   ~IPAR_config() override {
     if (!m_temp_folder.empty()) {
       shcore::remove_directory(m_temp_folder);
-    };
+    }
   }
 
   const PAR_structure &par() const { return m_par; }

mysqlshdk/libs/storage/backend/oci_par_directory.cc

@@ -25,6 +25,9 @@
 
 #include <algorithm>
 #include <filesystem>
+#include <memory>
+#include <unordered_set>
+#include <utility>
 
 #include "mysqlshdk/libs/db/uri_encoder.h"
 #include "mysqlshdk/libs/oci/oci_par.h"
@@ -161,8 +164,8 @@ std::string Oci_par_directory::get_list_url() const {
   // files in the current directory
   std::string url = "?fields=name,size&delimiter=/";
 
-  if (!m_config->par().object_prefix.empty()) {
-    url += "&prefix=" + pctencode_query_value(m_config->par().object_prefix);
+  if (!m_config->par().object_prefix().empty()) {
+    url += "&prefix=" + pctencode_query_value(m_config->par().object_prefix());
   }
 
   if (!m_next_start_with.empty()) {
@@ -179,7 +182,7 @@ std::unordered_set<IDirectory::File_info> Oci_par_directory::parse_file_list(
   const auto response = shcore::Value::parse(data.data(), data.size()).as_map();
   const auto objects = response->get_array("objects");
   m_next_start_with = response->get_string("nextStartWith", "");
-  const auto prefix_length = m_config->par().object_prefix.length();
+  const auto prefix_length = m_config->par().object_prefix().length();
 
   list.reserve(objects->size());
 

unittest/modules/util/dump/dump_manifest_t.cc

@@ -1,4 +1,4 @@
-/* Copyright (c) 2020, 2022, Oracle and/or its affiliates.
+/* Copyright (c) 2020, 2023, Oracle and/or its affiliates.
 
  This program is free software; you can redistribute it and/or modify
  it under the terms of the GNU General Public License, version 2.0,
@@ -295,34 +295,14 @@ TEST_F(Oci_os_tests, dump_manifest_read_mode) {
   EXPECT_STREQ(k_objects[Obj_index::SECOND].content.c_str(),
                second_file_data.c_str());
 
-  // Test the read manifest allows writing to file using Read/Write PAR
-  auto rw_par = bucket.create_pre_authenticated_request(
+  // any file which does not belong to the manifest results in an exception
+  const auto rw_par = bucket.create_pre_authenticated_request(
       mysqlshdk::oci::PAR_access_type::OBJECT_READ_WRITE, time, "par-progress",
       "@.load.progress.json");
   const auto progress_file_uri =
       write_config->service_endpoint() + rw_par.access_uri;
-  auto new_progress_file = read_manifest.file(progress_file_uri);
-
-  // Being a new file created with PAR it should NOT exist
-  EXPECT_FALSE(new_progress_file->exists());
-  EXPECT_THROW_LIKE(new_progress_file->file_size(), shcore::Exception,
-                    "Not Found");
-
-  new_progress_file->open(mysqlshdk::storage::Mode::WRITE);
-  new_progress_file->write("MY PROGRESS DATA", 16);
-  new_progress_file->close();
-
-  auto existing_progress_file = read_manifest.file(progress_file_uri);
-
-  // Being an existing file created with PAR it should exist
-  EXPECT_TRUE(existing_progress_file->exists());
-
-  existing_progress_file->open(mysqlshdk::storage::Mode::READ);
-
-  read = existing_progress_file->read(&buffer, 16);
-  std::string progress_data(buffer, read);
-  existing_progress_file->close();
-  EXPECT_STREQ("MY PROGRESS DATA", progress_data.c_str());
+  EXPECT_THROW_LIKE(read_manifest.file(progress_file_uri), shcore::Exception,
+                    "Unknown object in manifest");
 }
 
 }  // namespace testing

unittest/scripts/auto/py_oci/scripts/oci_prefix_par_dump_norecord.py

@@ -85,6 +85,10 @@ def anonymize_par(par, full=True):
     "bucket": create_par(OS_NAMESPACE, OS_BUCKET_NAME, "AnyObjectReadWrite", "all-read-par", today_plus_days(1, RFC3339), None, "ListObjects"),
 }
 
+# BUG#35548572 - PARs using dedicated endpoints
+for par_type, par in list(good_pars.items()):
+    good_pars[par_type + "-converted"] = convert_par(par)
+
 #@<> Testing PAR with conflicting options
 for par_type, par in good_pars.items():
     for name, callback in dump_util_cb.items():
@@ -150,6 +154,8 @@ def anonymize_par(par, full=True):
 #<> Testing dump attempt, with existing data in the target bucket/prefix
 for par_type, par in good_pars.items():
     # Performs an initial dump, to ensure there's data in the target location for the following tests
+    print(f"--> Preparing util.dump_{name} test with existing files in target location using {par_type} PAR")
+    prepare_empty_bucket(OS_BUCKET_NAME, OS_NAMESPACE, False)
     util.dump_tables("sakila", ["category"], par)
     for name, callback in dump_util_cb.items():
         print(f"--> Testing util.dump_{name} using {par_type} PAR with existing files in target location")
@@ -165,6 +171,7 @@ def anonymize_par(par, full=True):
         shell.connect(__sandbox_uri1)
         EXPECT_NO_THROWS(lambda: callback(par, None), f"Unexpected error calling util.dump_{name} with {par_type} PAR")
         session.close()
+        print(f"--> Loading dump created by util.dump_{name} using {par_type} PAR")
         shell.connect(__sandbox_uri2)
         PREPARE_PAR_IS_SECRET_TEST()
         EXPECT_NO_THROWS(lambda: util.load_dump(par, {"progressFile":"progress.txt"}), f"Unexpected error loading dump using {par_type} PAR.")

unittest/scripts/auto/py_oci/scripts/oci_utils.inc

@@ -2,6 +2,7 @@ import oci
 import datetime
 import time
 import queue
+import re
 import threading
 
 from oci.object_storage.models.create_preauthenticated_request_details import CreatePreauthenticatedRequestDetails
@@ -189,3 +190,14 @@ def EXPECT_PAR_IS_SECRET():
     check_text(testutil.fetch_captured_stdout(False), "stdout")
     check_text(testutil.fetch_captured_stderr(False), "stderr")
     check_text(testutil.cat_file(testutil.get_shell_log_path()), "Shell log")
+
+def convert_par(par):
+    m = re.match(r"^https:\/\/(?:([^\.]+)\.)?objectstorage\.([^\.]+)\.[^\/]+(\/p\/.+\/n\/(.+)\/b\/.*\/o\/(?:(?:.*)\/)?.*)$", par)
+    if m is None:
+        raise Exception(f"This is not a PAR: {par}")
+    if m.group(1) is None:
+        # old format, return an URL with a dedicated endpoint
+        return f"https://{m.group(4)}.objectstorage.{m.group(2)}.oci.customer-oci.com{m.group(3)}"
+    else:
+        # PAR with a dedicated endpoint, return old format
+        return f"https://objectstorage.{m.group(2)}.oraclecloud.com{m.group(3)}"