Sindbad~EG File Manager

"""
This program is free software: you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License,
or (at your option) any later version.


This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 
See the GNU General Public License for more details.


You should have received a copy of the GNU General Public License
 along with this program.  If not, see <https://www.gnu.org/licenses/>.

Copyright © 2019 Cloud Linux Software Inc.

This software is also available under ImunifyAV commercial license,
see <https://www.imunify360.com/legal/eula>
"""
import asyncio
import json
import logging
import pwd
import shutil
import time
from abc import ABC
from dataclasses import dataclass
from pathlib import Path
from typing import List, Optional

import psutil

from defence360agent.contracts.messages import Message, MessageType
from defence360agent.internals.global_scope import g
from defence360agent.internals.the_sink import TheSink
from defence360agent.subsys.panels import hosting_panel
from defence360agent.utils import rmtree
from defence360agent.utils.threads import to_thread
from imav.malwarelib.config import (
    ExitDetachedScanType,
    MalwareScanResourceType,
)
from imav.malwarelib.scan import (
    ScanAlreadyCompleteError,
    ScanInfoError,
)
from imav.malwarelib.scan.detached import (
    DetachedDir,
    DetachedOperation,
    DetachedScan,
    DetachedState,
)
from imav.malwarelib.scan.mds import MDS, MDS_PATH
from imav.malwarelib.scan.mds.report import (
    MalwareDatabaseHitInfo,
    scan_report,
)
from imav.malwarelib.scan.utils import trim_file_content

MDS_PID_WAIT_TIME = 30

logger = logging.getLogger(__name__)


class DetachedOperationFailed(Exception):
    pass


class MDSDetachedScanDir(DetachedDir):
    DETACHED_DIR = "/var/imunify360/dbscan/run/scan"

    ignore_file: Path
    report_file: Path
    scan_info_file: Path

    FILES = {
        **DetachedDir.FILES,
        "ignore_file": "ignore",
        "report_file": "report_file",
        "scan_info_file": "scan_info.json",
    }


class MDSDetachedCleanupDir(DetachedDir):
    DETACHED_DIR = "/var/imunify360/dbscan/run/clean"

    report_file: Path

    FILES = {
        **DetachedDir.FILES,
        "report_file": "report_file",
    }


class MDSDetachedRestoreDir(DetachedDir):
    DETACHED_DIR = "/var/imunify360/dbscan/run/restore"

    report_file: Path

    FILES = {
        **DetachedDir.FILES,
        "report_file": "report_file",
    }


@dataclass
class DbScanInfo:
    cmd: List[str]
    scan_type: str = None
    initial_path: str = None
    started: int = 0


class MDSDetachedOperation(DetachedOperation, ABC):
    def get_detached_process_state(self) -> str:
        if not self.detached_dir.is_dir():
            logger.debug("No such scan directory: %s", self.detached_dir)
            return DetachedState.NO_DIR

        if self.detached_dir.done_file.is_file():
            return DetachedState.FINISHED

        try:
            pid = self.get_pid()
        except (FileNotFoundError, ValueError) as exc:
            logger.warning("Can't get pid for the detached operation: %s", exc)
            return DetachedState.ABORTED

        try:
            proc = psutil.Process(pid)
            if self.process_is_suitable(proc):
                return DetachedState.RUNNING
            else:  # not suitable, some new process with the same pid running
                logger.warning(
                    "The running process (pid=%s) does not match "
                    "the detached operation",
                    pid,
                )
        except psutil.Error as exc:
            logger.warning(
                "Can't get the process for the detached operation"
                " (pid=%s): %s",
                pid,
                exc,
            )

        return DetachedState.ABORTED


class MDSDetachedScan(MDSDetachedOperation, DetachedScan):
    RESOURCE_TYPE = MalwareScanResourceType.DB
    DETACHED_DIR_CLS = MDSDetachedScanDir

    def _is_scan_finished(self):
        return self.detached_dir.done_file.exists()

    def _get_progress_info(self):
        try:
            with self.detached_dir.progress_file.open() as fp:
                return json.load(fp)
        except (FileNotFoundError, json.JSONDecodeError):
            return {}

    @property
    def progress(self):
        progress_info = self._get_progress_info()
        try:
            return int(float(progress_info["progress_main"]))
        except KeyError:
            return 100 if self._is_scan_finished() else 0

    @property
    def phase(self):
        if self.progress == 0:
            return "avd scanning"
        return "{} scanning".format(MDS)

    @property
    def total_resources(self):
        try:
            return int(self._get_progress_info()["total_db_count"])
        except KeyError:
            return 0

    def _load_scan_info(self) -> Optional[DbScanInfo]:
        try:
            with self.detached_dir.scan_info_file.open() as fp:
                info = json.load(fp)
            started = self.detached_dir.path.stat().st_mtime
        except (FileNotFoundError, json.JSONDecodeError):
            return None
        return DbScanInfo(
            cmd=info["cmd"],
            scan_type=info["scan_type"],
            initial_path=info["initial_path"],
            started=started,
        )

    def _load_single_report(self, report_file):
        with report_file.open() as f:
            return json.load(f)

    @classmethod
    def process_is_suitable(cls, proc) -> bool:
        if proc:
            cmdline = proc.cmdline()
            return MDS_PATH in cmdline and "--scan" in cmdline
        return False

    async def complete(self) -> MessageType.MalwareDatabaseScan:
        if (scan_info := self._load_scan_info()) is None:
            raise ScanInfoError

        message = MessageType.MalwareDatabaseScan(
            args=scan_info.cmd,
            path=scan_info.initial_path,
            scan_id=self.detached_id,
            type=scan_info.scan_type,
        )

        try:
            hit_report_list = [
                self._load_single_report(report_file)
                for report_file in self.detached_dir.path.glob("report*.json")
            ]
        except FileNotFoundError as e:
            raise ScanAlreadyCompleteError from e
        except Exception as e:
            logger.exception("Unable to parse MDS report")
            message.update_with_error(str(e))
            return message

        if not hit_report_list:
            message.update_with_error("No reports found")
            return message

        for report in hit_report_list:
            report_msg = MessageType.MDSReport(
                report, scan_id=self.detached_id
            )
            await g.sink.process_message(report_msg)

        result_report = await scan_report(hit_report_list)
        message.update_with_report(result_report)
        return message

    async def kill_running_scan_process(self, timer=time.monotonic):
        error = None
        deadline = timer() + MDS_PID_WAIT_TIME
        while timer() < deadline:
            try:
                pid = self.get_pid()
                break
            except (FileNotFoundError, ValueError) as err:
                await asyncio.sleep(1)
                error = err
        else:
            logger.warning(
                "Cannot find the mds process to kill (%s): %r."
                " Assuming it's already dead.",
                self.detached_id,
                error,
            )
            return
        try:
            proc = psutil.Process(pid)
            if self.process_is_suitable(proc):
                proc.kill()
        except psutil.Error as err:
            logger.warning(
                "Problem when killing the running mds process: %s", err
            )

    async def handle_aborted_process(
        self,
        *,
        sink,
        exit_type: str = ExitDetachedScanType.ABORTED,
        kill: bool = True,
        scan_path: Optional[str] = None,
        scan_type: Optional[str] = None,
        scan_started: Optional[float] = None,
        cmd: Optional[List[str]] = None,
        out: str = "",
        err: str = "",
    ) -> None:
        """Removes aborted detached scan from scan_queue and writes it to DB.

        - Parses data about scan from scan_queue and writes it to DB
        - Kills scan process, if it exists
        - Deletes scan_dir
        - Processes MalwareScan and ScanFailed(in case of 'ABORTED') messages

        :param sink: the sink to send messages
        :param exit_type: 'ABORTED' by default,
            if stopped by user, then 'STOPPED'
        :param kill: try to kill a process
        :param scan_path: which path was scanned
        :param scan_type: what is the scan's type
        :param scan_started: when was the scan started (if known)
        :param cmd: command line arguments
        :param out: command stdout
        :param err: command stderr
        """
        if kill:
            await self.kill_running_scan_process()
        scan_info = self._load_scan_info() or DbScanInfo(cmd=[])
        cmd = cmd or scan_info.cmd
        scan_path = scan_path or scan_info.initial_path
        scan_type = scan_type or scan_info.scan_type
        scan_started = int(scan_started or scan_info.started)
        await sink.process_message(
            MessageType.MalwareDatabaseScan(
                args=cmd,
                error=exit_type,
                path=scan_path,
                scan_id=self.detached_id,
                type=scan_type,
                started=scan_started,
                completed=int(time.time()),
            )
        )

        scan_dir = self.detached_dir
        if exit_type == ExitDetachedScanType.ABORTED:
            stdout = trim_file_content(scan_dir.log_file) or out
            stderr = trim_file_content(scan_dir.err_file) or err
            msg = MessageType.ScanFailed()
            msg["out"] = stdout
            msg["err"] = stderr
            logger.warning(
                "Scan %s was aborted: %s, %s", self.detached_id, stdout, stderr
            )
            msg["command"] = cmd
            msg["message"] = "aborted"
            msg["scan_id"] = self.detached_id
            msg["path"] = scan_path
            await sink.process_message(msg)

        if not scan_dir.path.is_dir():
            logger.warning("No such directory: %s", scan_dir)
        else:
            rmtree(str(scan_dir))


class MDSDetachedMutableOperation(MDSDetachedOperation, ABC):
    """Parsing of operations that can succeed or fail for any DB hit"""

    SUCCESS_MSG = None
    FAIL_MSG = None

    async def complete(self) -> Message:
        message = {}
        try:
            hit_report_list = [
                self._load_single_report(report)
                for report in self.detached_dir.path.glob("report*.json")
            ]
            message = await self._parse_report(hit_report_list)
        except FileNotFoundError as e:
            raise ScanAlreadyCompleteError from e
        except DetachedOperationFailed as e:
            message["error"] = str(e)
            logger.exception(f"Unable to parse MDS {self.NAME} report")
            return self.FAIL_MSG(error=str(e))
        else:
            if not hit_report_list:
                return self.FAIL_MSG(error=f"No {self.NAME} reports found")

            for report in hit_report_list:
                report_msg = MessageType.MDSReport(
                    report, scan_id=self.detached_id
                )
                await g.sink.process_message(report_msg)

            return self.SUCCESS_MSG(**message)

    @staticmethod
    def _load_single_report(report_file):
        with report_file.open() as f:
            return json.load(f)

    async def _parse_report(self, hit_report_list) -> dict:
        users_from_panel = set(await hosting_panel.HostingPanel().get_users())
        pw_all = await to_thread(pwd.getpwall)

        succeeded = set()
        failed = set()
        for report in hit_report_list:
            if errors := report["error_list"]:
                logger.error(f"Errors in MDS {self.NAME}: %s", errors)
                raise DetachedOperationFailed(errors)
            if not report["rows_with_error"]:
                succeeded.add(
                    MalwareDatabaseHitInfo.from_report(
                        report, users_from_panel, pw_all
                    )
                )
            else:
                failed.add(
                    MalwareDatabaseHitInfo.from_report(
                        report, users_from_panel, pw_all
                    )
                )

        return {"succeeded": succeeded, "failed": failed}


class MDSDetachedCleanup(MDSDetachedMutableOperation):
    NAME = "cleanup"
    DETACHED_DIR_CLS = MDSDetachedCleanupDir
    SUCCESS_MSG = MessageType.MalwareDatabaseCleanup
    FAIL_MSG = MessageType.MalwareDatabaseCleanupFailed

    @property
    def on_complete_message(self) -> MessageType.MalwareCleanComplete:
        return MessageType.MalwareCleanComplete(
            scan_id=self.detached_id,
        )

    async def handle_aborted_process(
        self,
        *,
        sink: TheSink,
        exit_type: str = ExitDetachedScanType.ABORTED,
        scan_path: Optional[str] = None,
    ) -> None:
        assert (
            exit_type == ExitDetachedScanType.ABORTED
        ), "Cleanup cannot be stopped, only aborted status is supported"

        # NOTE: No need to kill running process because Imunify360 does
        # it through systemd and we don't support MDS for Imunify AV.
        logger.info("Cleanup %s was %s", self.detached_id, exit_type)

        stdout = trim_file_content(self.detached_dir.log_file)
        stderr = trim_file_content(self.detached_dir.err_file)
        msg = MessageType.MalwareDatabaseCleanupFailed(
            error=(
                f"path: {scan_path}, "
                f"detached_id: {self.detached_id}, "
                f"out: {stdout}, "
                f"err: {stderr}"
            )
        )
        await sink.process_message(msg)

        shutil.rmtree(str(self.detached_dir.path), ignore_errors=True)

    @classmethod
    def process_is_suitable(cls, proc) -> bool:
        if proc:
            cmdline = proc.cmdline()
            return MDS_PATH in cmdline and "--clean" in cmdline
        return False


class MDSDetachedRestore(MDSDetachedMutableOperation):
    NAME = "restore"
    DETACHED_DIR_CLS = MDSDetachedRestoreDir
    SUCCESS_MSG = MessageType.MalwareDatabaseRestore
    FAIL_MSG = MessageType.MalwareDatabaseRestoreFailed

    @classmethod
    def process_is_suitable(cls, proc) -> bool:
        if proc:
            cmdline = proc.cmdline()
            return MDS_PATH in cmdline and "--restore" in cmdline
        return False

    @property
    def on_complete_message(self) -> MessageType.MalwareRestoreComplete:
        return MessageType.MalwareRestoreComplete(
            scan_id=self.detached_id,
        )

    async def handle_aborted_process(
        self,
        *,
        sink: TheSink,
        exit_type: str = ExitDetachedScanType.ABORTED,
        scan_path: Optional[str] = None,
    ) -> None:
        assert (
            exit_type == ExitDetachedScanType.ABORTED
        ), "Restore cannot be stopped, only aborted status is supported"

        logger.info("Restore %s was %s", self.detached_id, exit_type)

        stdout = trim_file_content(self.detached_dir.log_file)
        stderr = trim_file_content(self.detached_dir.err_file)
        msg = MessageType.MalwareDatabaseRestoreFailed(
            error=(
                f"path: {scan_path}, "
                f"detached_id: {self.detached_id}, "
                f"out: {stdout}, "
                f"err: {stderr}"
            ),
        )
        await sink.process_message(msg)

        shutil.rmtree(str(self.detached_dir.path), ignore_errors=True)