Skip to content

[WIP] patch for discussion #174 #175

New issue

Have a question about this project? # for a free GitHub account to open an issue and contact its maintainers and the community.

#

By clicking “#”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? # to your account

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

[WIP] patch for discussion #174 #175

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2b1d186
Fix default "log_directory" value
mikecaat May 14, 2021
293fb41
Fix to support restoring server logs
mikecaat May 14, 2021
aff9389
Add regression tests for 293fb4167 and 2b1d18651
mikecaat May 14, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions expected/arc_srv_log_management.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,3 +44,24 @@ Number of existing server log files: 7
do --keep-srvlog-files=4 AND --keep-srvlog-days=1
0
Number of remaining server log files: 4
###### LOG FILE MANAGEMENT TEST-0007 ######
###### keep-srvlog-files only with default "log_directory" option ######
0
Number of existing server log files: 4
do --keep-srvlog-files=3
0
Number of remaining server log files: 3
###### LOG FILE MANAGEMENT TEST-0008 ######
###### keep-srvlog-days only with default "log_directory" option ######
0
Number of existing server log files: 7
do --keep-srvlog-days=1
0
Number of remaining server log files: 3
###### LOG FILE MANAGEMENT TEST-0009 ######
###### keep-srvlog-files and keep-srvlog-days together with default "log_directory" option ######
0
Number of existing server log files: 7
do --keep-srvlog-files=4 AND --keep-srvlog-days=1
0
Number of remaining server log files: 4
24 changes: 24 additions & 0 deletions expected/restore.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,3 +70,27 @@ OK: hard-copy option works well.
0
0

###### RESTORE COMMAND TEST-0013 ######
###### server logs (in the default log_directory) do not be backuped without "--with-serverlog" ######
0
0
1

###### RESTORE COMMAND TEST-0014 ######
###### server logs (in the default log_directory) should be backuped with "--with-serverlog" ######
0
0
0

###### RESTORE COMMAND TEST-0015 ######
###### server logs do not be backuped without "--with-serverlog" ######
0
0
1

###### RESTORE COMMAND TEST-0016 ######
###### server logs should be backuped with "--with-serverlog" ######
0
0
0

24 changes: 24 additions & 0 deletions expected/restore_checksum.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,3 +70,27 @@ OK: hard-copy option works well.
0
0

###### RESTORE COMMAND TEST-0013 ######
###### server logs (in the default log_directory) do not be backuped without "--with-serverlog" ######
0
0
1

###### RESTORE COMMAND TEST-0014 ######
###### server logs (in the default log_directory) should be backuped with "--with-serverlog" ######
0
0
0

###### RESTORE COMMAND TEST-0015 ######
###### server logs do not be backuped without "--with-serverlog" ######
0
0
1

###### RESTORE COMMAND TEST-0016 ######
###### server logs should be backuped with "--with-serverlog" ######
0
0
0

4 changes: 2 additions & 2 deletions init.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,9 +176,9 @@ do_init(void)
}
else if (pgdata)
{
/* default: log_directory = 'pg_log' */
/* default: log_directory = 'log' if PostgreSQL version is 10 or above */
srvlog_path = pgut_malloc(MAXPGPATH);
join_path_components(srvlog_path, pgdata, "pg_log");
join_path_components(srvlog_path, pgdata, "log");
}
}
if (srvlog_path)
Expand Down
116 changes: 116 additions & 0 deletions restore.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
static void backup_online_files(bool re_recovery);
static void restore_online_files(void);
static void restore_database(pgBackup *backup);
static void restore_server_logs(pgBackup *backup);
static void restore_archive_logs(pgBackup *backup, bool is_hard_copy);

static void add_recovery_related_options(const char *target_time,
Expand Down Expand Up @@ -276,6 +277,9 @@ do_restore(const char *target_time,

/* restore base backup */
restore_database(base_backup);

/* restore server logs */
restore_server_logs(base_backup);

last_restored_index = base_index;

Expand Down Expand Up @@ -308,6 +312,7 @@ do_restore(const char *target_time,
timestamp);

restore_database(backup);
restore_server_logs(backup);
last_restored_index = i;
}

Expand Down Expand Up @@ -612,6 +617,117 @@ restore_database(pgBackup *backup)
printf(_("restore backup completed\n"));
}

/*
* restore server logs
*/
void
restore_server_logs(pgBackup *backup)
{
char timestamp[100];
char path[MAXPGPATH];
char list_path[MAXPGPATH];
parray *files;
int i;
int num_skipped = 0;

/* we need completed serverlog backup */
if (!(backup->status == BACKUP_STATUS_OK && backup->with_serverlog))
return;

time2iso(timestamp, lengthof(timestamp), backup->start_time);
if (verbose && !check)
{
printf(_("----------------------------------------\n"));
}

/*
* Validate backup files with its size, because load of CRC calculation is
* not light.
*/
pgBackupValidate(backup, true, false, true);

if (backup->backup_mode == BACKUP_MODE_FULL)
elog(INFO, "restoring server log files from the full mode backup \"%s\"",
timestamp);
else if (backup->backup_mode == BACKUP_MODE_INCREMENTAL)
elog(INFO, "restoring server log files from the incremental mode backup \"%s\"",
timestamp);

/*
* get list of files which need to be restored.
*/
pgBackupGetPath(backup, path, lengthof(path), SRVLOG_DIR);
pgBackupGetPath(backup, list_path, lengthof(list_path), SRVLOG_FILE_LIST);
files = dir_read_file_list(path, list_path);
for (i = parray_num(files) - 1; i >= 0; i--)
{
pgFile *file = (pgFile *) parray_get(files, i);

/* remove files which are not backed up */
if (file->write_size == BYTES_INVALID)
pgFileFree(parray_remove(files, i));
}

/* restore server log files into $PGDATA */
for (i = 0; i < parray_num(files); i++)
{
char from_root[MAXPGPATH];
pgFile *file = (pgFile *) parray_get(files, i);

pgBackupGetPath(backup, from_root, lengthof(from_root), SRVLOG_DIR);

/* check for interrupt */
if (interrupted)
ereport(FATAL,
(errcode(ERROR_INTERRUPTED),
errmsg("interrupted during restore server log files")));

/* print progress in verbose mode */
if (verbose && !check)
printf(_("(%d/%lu) %s "), i + 1, (unsigned long) parray_num(files),
file->path + strlen(from_root) + 1);

/* not backed up */
if (file->write_size == BYTES_INVALID)
{
num_skipped++;
if (verbose && !check)
printf(_("not backed up, skip\n"));
goto show_progress;
}

/* restore a server log file */
if (!check)
copy_file(from_root, srvlog_path, file, backup->compress_data);

/* print size of restored file */
if (verbose && !check)
{
printf(_("restored %lu\n"), (unsigned long) file->write_size);
continue;
}

show_progress:
/* print progress in non-verbose format */
if (progress)
{
fprintf(stderr, _("Processed %d of %lu files, skipped %d"),
i + 1, (unsigned long) parray_num(files), num_skipped);
if(i + 1 < (unsigned long) parray_num(files))
fprintf(stderr, "\r");
else
fprintf(stderr, "\n");
}
}

/* cleanup */
parray_walk(files, pgFileFree);
parray_free(files);

if (verbose && !check)
printf(_("restore server log files completed\n"));
}

/*
* Restore archived WAL by creating symbolic link which linked to backup WAL in
* archive directory.
Expand Down
Loading