Commit a17591b1 authored by Nigel Kukard's avatar Nigel Kukard
Browse files

borg: Parse status info together with other output

parent 36168e9f
Pipeline #7155 passed with stage
in 47 seconds
......@@ -243,9 +243,6 @@ class BorgBackup: # noqa: R0902 pylint: disable=too-many-instance-attributes
try:
# Create process
with subprocess.Popen(**popen_args) as process: # nosec
# For a borg backup, the summary seems to be json split over lines, so we need to keep track once we have the stats
# the parse that chunk of data
summary_output = ""
# Check if the pipe_process is ok if we have one...
if pipe_process:
result_code_pp = pipe_process.poll()
......@@ -254,12 +251,7 @@ class BorgBackup: # noqa: R0902 pylint: disable=too-many-instance-attributes
raise BackupError(f"Error piping data from child process to Borg, exit code {result_code_pp}") from None
# Loop with output
for line in chunks(process.stdout, delim="\n\r"):
# Parse output if we don't have stats yet
if not stats:
self._parse_output(line, stats, stats_state)
# Once we have stats, save the rest in our summary...
else:
summary_output += line
self._parse_output(line, stats, stats_state)
# Grab return code
process.communicate()
result_code = process.poll()
......@@ -268,9 +260,6 @@ class BorgBackup: # noqa: R0902 pylint: disable=too-many-instance-attributes
if pipe_process:
pipe_process.communicate()
result_code_pp = pipe_process.poll()
# Lastly check if we have a summary_output, if so, try parse it
if summary_output:
self._parse_output_summary(summary_output, stats)
# Save result code
if pipe_process:
self.notifiers.info(f"Borg completed: status={result_code}, pipe status={result_code_pp}")
......@@ -329,33 +318,6 @@ class BorgBackup: # noqa: R0902 pylint: disable=too-many-instance-attributes
return uri
def _parse_output_summary(self, chunk: str, stats: BackstepStats) -> None:
"""Parse output chunk from borg, which should be our summary."""
# Make sure the chunk is json
if chunk.startswith("{") and chunk.endswith("}"):
# Try decode json
try:
chunk_json = json.loads(chunk)
except json.JSONDecodeError as err:
self.notifiers.warning(f"Failed to decode JSON output summary: {err}\n{chunk}")
return
# Store stats
stats["duration"] = int(chunk_json["archive"]["duration"])
stats["backup_size"] = chunk_json["archive"]["stats"]["original_size"]
stats["backup_size_compressed"] = chunk_json["archive"]["stats"]["compressed_size"]
stats["backup_size_deduplicated"] = chunk_json["archive"]["stats"]["deduplicated_size"]
stats["backup_size_actual"] = stats["backup_size_deduplicated"]
stats["backup_ratio"] = f'{(stats["backup_size_actual"] / stats["backup_size"]) * 100:.2f}'
stats["total_size"] = chunk_json["cache"]["stats"]["total_size"]
stats["total_size_compressed"] = chunk_json["cache"]["stats"]["total_csize"]
stats["total_size_deduplicated"] = chunk_json["cache"]["stats"]["unique_csize"]
stats["total_size_actual"] = stats["total_size_deduplicated"]
stats["total_ratio"] = f'{(stats["total_size_actual"] / stats["total_size"]) * 100:.2f}'
else:
self.notifiers.warning(f"Chunk not understood from borg: {chunk}")
def _parse_output( # noqa: C901 pylint: disable=too-many-branches,too-many-statements
self, line: str, stats: BackstepStats, stats_state: Dict[str, Any]
) -> None:
......@@ -389,8 +351,23 @@ class BorgBackup: # noqa: R0902 pylint: disable=too-many-instance-attributes
self, json_obj: Dict[str, Any], stats: BackstepStats, stats_state: Dict[str, Any]
) -> None:
# Check if this is the archive information message
if "archive" in json_obj:
# Store stats
stats["duration"] = int(json_obj["archive"]["duration"])
stats["backup_size"] = json_obj["archive"]["stats"]["original_size"]
stats["backup_size_compressed"] = json_obj["archive"]["stats"]["compressed_size"]
stats["backup_size_deduplicated"] = json_obj["archive"]["stats"]["deduplicated_size"]
stats["backup_size_actual"] = stats["backup_size_deduplicated"]
stats["backup_ratio"] = f'{(stats["backup_size_actual"] / stats["backup_size"]) * 100:.2f}'
stats["total_size"] = json_obj["cache"]["stats"]["total_size"]
stats["total_size_compressed"] = json_obj["cache"]["stats"]["total_csize"]
stats["total_size_deduplicated"] = json_obj["cache"]["stats"]["unique_csize"]
stats["total_size_actual"] = stats["total_size_deduplicated"]
stats["total_ratio"] = f'{(stats["total_size_actual"] / stats["total_size"]) * 100:.2f}'
# Check for generic progress messages
if (json_obj["type"] == "progress_message") or (json_obj["type"] == "progress_percent"):
elif (json_obj["type"] == "progress_message") or (json_obj["type"] == "progress_percent"):
# Line key for the last line we ignored
msgid_line = f'last_line_{json_obj["msgid"]}'
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment