Bug#27571663 - MEB FAILS WHEN INNODB_LOG_FILE_SIZE IS DIFFERENT FOR INITIALIZE AND SERVER START

MEB failed do scan the redo log up to its logical end under
circumstances. One such case was resizing of the log files.
Resizing can happen by shutdown and start of the server with
a new value for the log file size. The case was repeatable
after two resizes with one megabyte of extended size each.

The core of the problem lies in a new redo log format
(version 3). In this format, the log header field
LOG_HEADER_START_LSN does not always have the expected
value. Instead the relation of LOG_CHECKPOINT_LSN and
LOG_CHECKPOINT_OFFSET is significant for lsn to offset
calculations.

So far MEB relied on LOG_CHECKPOINT_LSN for lsn to offset
calculations. Due to the unreliable values, MEB could start
to read the log from a wrong offset. It could get a
zero-filled block, on which it assumed end of log.

This patch fixes the lsn to offset calculations. It gets rid
of MEB's own calculation and calls an InnoDB function for
this calculation instead. (log_files_real_offset_for_lsn()
from log0write.cc, log0log.h, log0types.h).

It does also fix the creation of the log files at restore
time. The checkpoint offset field needs to be set
appropriately.

Unrelated changes:

Fix variable types used for lsns and offsets. Lsns have
lsn_t and offsets have utin64_t.

Reorder functions around the calculation in header and
implementation file to a consistent order.

Reduce the number of functions with similar signature.

Rename variables, so that the same name is used for the
same "thing" throughout.

Rename functions so, that a prefix of log_ is reserved for
InnoDB functions, a prefix of meb_ is used for MEB
functions, that are implemented inside InnoDB sources, a
function name with get_ extracts information from memory
structures, a function name with read_ invloves reading of
information from a file.

Move meb_read_checkpoint_info from log0recv.cc to
Redo_log_adapter::get_checkpoint_info().

Fix a message about parsing redo log when redo log was not
parsed.

Fix tests, that used the message about parsing. Use the
message about scanning instead.

In tests, use $MYSQLTEST_VARDIR/tmp for logs.
$MYSQLTEST_VARDIR/log is not preserved by MTR after a
failing test. But the MEB logs are most valuable then.

Add new test script meb.log_size_increase to repeat the
reported bug when the patch is not applied.

RB#19145

Author: Ingo Struewing

storage/innobase/include/log0log.h

@@ -853,6 +853,7 @@ void log_buffer_x_lock_enter(log_t &log);
 /** Releases the log buffer x-lock.
 @param[in,out]	log	redo log */
 void log_buffer_x_lock_exit(log_t &log);
+#endif /* !UNIV_HOTBACKUP */
 
 /** Calculates offset within log files, excluding headers of log files.
 @param[in]	log		redo log
@@ -872,6 +873,7 @@ for the provided lsn value.
 @param[in]	lsn	log sequence number
 @return real offset within the log files */
 uint64_t log_files_real_offset_for_lsn(const log_t &log, lsn_t lsn);
+#ifndef UNIV_HOTBACKUP
 
 /** Changes size of the log buffer. This is a thread-safe version.
 It is used by SET GLOBAL innodb_log_buffer_size = X.

storage/innobase/include/log0recv.h

@@ -65,17 +65,6 @@ extern volatile lsn_t backup_redo_log_flushed_lsn;
 /** TRUE when the redo log is being backed up */
 extern bool recv_is_making_a_backup;
 
-/** Reads the checkpoint info needed in hot backup.
-@param[in]	hdr		buffer containing the log group header
-@param[out]	lsn		checkpoint lsn
-@param[out]	offset		checkpoint offset in the log group
-@param[out]	cp_no		checkpoint number
-@param[out]	first_header_lsn lsn of of the start of the first log file
-@return true if success */
-bool meb_read_checkpoint_info(const byte *hdr, lsn_t *lsn, lsn_t *offset,
-                              lsn_t *cp_no, lsn_t *first_header_lsn)
-    MY_ATTRIBUTE((warn_unused_result));
-
 /** Scans the log segment and n_bytes_scanned is set to the length of valid
 log scanned.
 @param[in]	buf			buffer containing log data

storage/innobase/include/log0types.h

@@ -326,6 +326,7 @@ struct alignas(INNOBASE_CACHE_LINE_SIZE) log_t {
   /** Up to this file offset in the log files, the write-ahead
   has been done or is not required (for any other reason). */
   uint64_t write_ahead_end_offset;
+#endif /* !UNIV_HOTBACKUP */
 
   /** Some lsn value within the current log file. */
   lsn_t current_file_lsn;
@@ -347,6 +348,7 @@ struct alignas(INNOBASE_CACHE_LINE_SIZE) log_t {
   including headers of the log files. */
   uint64_t files_real_capacity;
 
+#ifndef UNIV_HOTBACKUP
   /** Mutex which can be used to pause log writer thread. */
   ib_mutex_t writer_mutex;
 
@@ -465,10 +467,12 @@ struct alignas(INNOBASE_CACHE_LINE_SIZE) log_t {
 
   /** Lsn from which recovery has been started. */
   lsn_t recovered_lsn;
+#endif /* !UNIV_HOTBACKUP */
 
   /** Number of log files. */
   uint32_t n_files;
 
+#ifndef UNIV_HOTBACKUP
   /** Format of the redo log: e.g., LOG_HEADER_FORMAT_CURRENT. */
   uint32_t format;
 

storage/innobase/log/log0recv.cc

@@ -1299,48 +1299,6 @@ void recv_apply_hashed_log_recs(log_t &log, bool allow_ibuf) {
 }
 
 #else /* !UNIV_HOTBACKUP */
-/** Reads the checkpoint info needed in hot backup.
-@param[in]	hdr		buffer containing the log group header
-@param[out]	lsn		checkpoint lsn
-@param[out]	offset		checkpoint offset in the log group
-@param[out]	cp_no		checkpoint number
-@param[out]	first_header_lsn lsn of of the start of the first log file
-@return true if success */
-bool meb_read_checkpoint_info(const byte *hdr, lsn_t *lsn, lsn_t *offset,
-                              lsn_t *cp_no, lsn_t *first_header_lsn) {
-  ulint max_cp = 0;
-  uint64_t max_cp_no = 0;
-  const byte *cp_buf = hdr + LOG_CHECKPOINT_1;
-
-  if (recv_check_log_header_checksum(cp_buf)) {
-    max_cp_no = mach_read_from_8(cp_buf + LOG_CHECKPOINT_NO);
-    max_cp = LOG_CHECKPOINT_1;
-  }
-
-  cp_buf = hdr + LOG_CHECKPOINT_2;
-
-  if (recv_check_log_header_checksum(cp_buf)) {
-    if (mach_read_from_8(cp_buf + LOG_CHECKPOINT_NO) > max_cp_no) {
-      max_cp = LOG_CHECKPOINT_2;
-    }
-  }
-
-  if (max_cp == 0) {
-    return (false);
-  }
-
-  cp_buf = hdr + max_cp;
-
-  *lsn = mach_read_from_8(cp_buf + LOG_CHECKPOINT_LSN);
-  *offset = mach_read_from_8(cp_buf + LOG_CHECKPOINT_OFFSET);
-
-  *cp_no = mach_read_from_8(cp_buf + LOG_CHECKPOINT_NO);
-
-  *first_header_lsn = mach_read_from_8(hdr + LOG_HEADER_START_LSN);
-
-  return (true);
-}
-
 /** Scans the log segment and n_bytes_scanned is set to the length of valid
 log scanned.
 @param[in]	buf			buffer containing log data

storage/innobase/log/log0write.cc

@@ -46,8 +46,10 @@ this program; if not, write to the Free Software Foundation, Inc.,
 #include "buf0flu.h"
 #include "dict0boot.h"
 #include "dict0stats_bg.h"
+#endif /* !UNIV_HOTBACKUP */
 #include "fil0fil.h"
 #include "log0log.h"
+#ifndef UNIV_HOTBACKUP
 #include "log0recv.h"
 #include "mem0mem.h"
 #include "mysqld.h" /* server_uuid */
@@ -872,15 +874,19 @@ Wait_stats log_write_up_to(log_t &log, lsn_t end_lsn, bool flush_to_disk) {
   }
 }
 
-/* @} */
+  /* @} */
 
-/**************************************************/ /**
+  /**************************************************/ /**
 
- @name Log writer thread
+   @name Log writer thread
 
- *******************************************************/
+   *******************************************************/
 
-/* @{ */
+  /* @{ */
+
+#else /* !UNIV_HOTBACKUP */
+#define log_writer_mutex_own(log) true
+#endif /* !UNIV_HOTBACKUP */
 
 uint64_t log_files_size_offset(const log_t &log, uint64_t offset) {
   ut_ad(log_writer_mutex_own(log));
@@ -923,6 +929,7 @@ uint64_t log_files_real_offset_for_lsn(const log_t &log, lsn_t lsn) {
 
   return (log_files_real_offset(log, size_offset));
 }
+#ifndef UNIV_HOTBACKUP
 
 void log_files_update_offsets(log_t &log, lsn_t lsn) {
   ut_ad(log_writer_mutex_own(log));