submit

Includes code for generating the Job object, validating the object using Pydantic, and submitting the job.

Primarily intended for use with the hpcman queue submit command, but can be called from other places as well, for e.g. submitting specific, common job types to the cluster.

Currently, I expect to submit other jobs such as jupyterlab servers and gbrowse servers as well.

ByteSizeGnu

Bases: ByteSize

Modify ByteSize class to allow for Gnu-style byte sizes.

SGE parameters expect e.g. 4G, not 4GB, which is produced by human_readable() by default. This class overrides human_readable() to allow for Gnu-style byte sizes.

Source code in hpcman/queue/submit.py

class ByteSizeGnu(ByteSize):  # type: ignore
    """Modify ByteSize class to allow for Gnu-style byte sizes.

    SGE parameters expect e.g. `4G`, not `4GB`, which is produced by `human_readable()` by default. This class
    overrides `human_readable()` to allow for Gnu-style byte sizes.
    """

    def human_readable(self, decimal: bool = False, gnu: bool = False, slurm: bool = False) -> str:
        """Produces a human-readable byte size string.

        Args:
            decimal: Uses base 10 rather than base 2. Defaults to False.
            gnu: Produces e.g. `4G` rather than `4GB`. Defaults to False.

        Returns:
            str: Human-readable byte size string.
        """
        divisor = 1024
        units = "B", "KiB", "MiB", "GiB", "TiB", "PiB"
        final_unit = "EiB"
        if decimal:
            divisor = 1000
            units = "B", "KB", "MB", "GB", "TB", "PB"
            final_unit = "EB"
        if gnu:
            units = "B", "K", "M", "G", "T", "P"
            final_unit = "E"

        num = float(self)
        for unit in units:
            if abs(num) < divisor:
                if unit == "B" or slurm:
                    return f"{num:0.0f}{unit}"
                else:
                    return f"{num:0.1f}{unit}"
            num /= divisor
        if slurm:
            return f"{num:0.0f}{final_unit}"
        else:
            return f"{num:0.1f}{final_unit}"

human_readable(decimal=False, gnu=False, slurm=False)

Produces a human-readable byte size string.

Parameters:

Name	Type	Description	Default
decimal	bool	Uses base 10 rather than base 2. Defaults to False.	False
gnu	bool	Produces e.g. 4G rather than 4GB. Defaults to False.	False

Returns:

Name	Type	Description
str	str	Human-readable byte size string.

Source code in hpcman/queue/submit.py

def human_readable(self, decimal: bool = False, gnu: bool = False, slurm: bool = False) -> str:
    """Produces a human-readable byte size string.

    Args:
        decimal: Uses base 10 rather than base 2. Defaults to False.
        gnu: Produces e.g. `4G` rather than `4GB`. Defaults to False.

    Returns:
        str: Human-readable byte size string.
    """
    divisor = 1024
    units = "B", "KiB", "MiB", "GiB", "TiB", "PiB"
    final_unit = "EiB"
    if decimal:
        divisor = 1000
        units = "B", "KB", "MB", "GB", "TB", "PB"
        final_unit = "EB"
    if gnu:
        units = "B", "K", "M", "G", "T", "P"
        final_unit = "E"

    num = float(self)
    for unit in units:
        if abs(num) < divisor:
            if unit == "B" or slurm:
                return f"{num:0.0f}{unit}"
            else:
                return f"{num:0.1f}{unit}"
        num /= divisor
    if slurm:
        return f"{num:0.0f}{final_unit}"
    else:
        return f"{num:0.1f}{final_unit}"

ConfirmJobDir

Bases: PromptBase[JobPromptResponse]

A yes / no / other confirmation prompt.

We needed to edit the PromptBase because a boolean prompt would not be enough for these questions. The options for this prompt include standard y/n prompt, but also d/e/o (new) for SGE_Batch compatibility.

Source code in hpcman/queue/submit.py

class ConfirmJobDir(PromptBase[JobPromptResponse]):  # type: ignore
    """A yes / no / other confirmation prompt.

    We needed to edit the PromptBase because a boolean prompt would not be enough for these questions. The options for
    this prompt include standard y/n prompt, but also d/e/o (new) for `SGE_Batch` compatibility.
    """

    response_type = JobPromptResponse
    validate_error_message = (
        "[prompt.invalid]Please enter "
        "[green]y or d for yes,[/green] "
        "[yellow]o for new,[/yellow] or "
        "[red]n or e for no.[/red]"
    )

    choices: Dict[str, Tuple[str, ...]] = {
        "yes": ("y", "d"),
        "new": ("o",),
        "no": ("n", "e"),
    }
    default: TextType = "n"

    def make_prompt(self, default: DefaultType) -> Text:  # pyright: ignore
        """Make prompt text for three options
        Args:
            default (TextType): Default value.
        Returns:
            Text: Text to display in prompt.
        """
        prompt = self.prompt.copy()
        prompt.end = ""

        if self.show_choices and self.choices:
            yes, new, no = self.choices.values()
            _choices = " / ".join(
                [
                    f"yes: {','.join(yes)}",
                    f"new: {','.join(new)}",
                    f"no: {','.join(no)}",
                ]
            )
            choices = f"[{_choices}]"
            prompt.append(" ")
            prompt.append(choices, "prompt.choices")

        if self.default != ... and self.show_default and isinstance(self.default, (str, self.response_type)):
            prompt.append(" ")
            _default = self.render_default(self.default)
            prompt.append(_default)

        prompt.append(self.prompt_suffix)

        return prompt

    def process_response(self, value: str) -> JobPromptResponse:
        """Convert choices to JobPromptResponse."""
        value = value.strip().lower()
        valid_choices = [y for x in self.choices.values() for y in x]
        if value not in valid_choices:
            raise InvalidResponse(self.validate_error_message)
        if value in self.choices["yes"]:
            return JobPromptResponse.YES
        elif value in self.choices["new"]:
            return JobPromptResponse.NEW
        else:
            return JobPromptResponse.NO

make_prompt(default)

Make prompt text for three options

Parameters:

Name	Type	Description	Default
default	TextType	Default value.	required

Returns:

Name	Type	Description
Text	Text	Text to display in prompt.

Source code in hpcman/queue/submit.py

def make_prompt(self, default: DefaultType) -> Text:  # pyright: ignore
    """Make prompt text for three options
    Args:
        default (TextType): Default value.
    Returns:
        Text: Text to display in prompt.
    """
    prompt = self.prompt.copy()
    prompt.end = ""

    if self.show_choices and self.choices:
        yes, new, no = self.choices.values()
        _choices = " / ".join(
            [
                f"yes: {','.join(yes)}",
                f"new: {','.join(new)}",
                f"no: {','.join(no)}",
            ]
        )
        choices = f"[{_choices}]"
        prompt.append(" ")
        prompt.append(choices, "prompt.choices")

    if self.default != ... and self.show_default and isinstance(self.default, (str, self.response_type)):
        prompt.append(" ")
        _default = self.render_default(self.default)
        prompt.append(_default)

    prompt.append(self.prompt_suffix)

    return prompt

process_response(value)

Convert choices to JobPromptResponse.

Source code in hpcman/queue/submit.py

def process_response(self, value: str) -> JobPromptResponse:
    """Convert choices to JobPromptResponse."""
    value = value.strip().lower()
    valid_choices = [y for x in self.choices.values() for y in x]
    if value not in valid_choices:
        raise InvalidResponse(self.validate_error_message)
    if value in self.choices["yes"]:
        return JobPromptResponse.YES
    elif value in self.choices["new"]:
        return JobPromptResponse.NEW
    else:
        return JobPromptResponse.NO

Job

Bases: BaseModel

Job class based on Pydantic BaseModel.

Raises:

Type	Description
ValueError	When invalid inputs are provided.

Attributes:

Name	Type	Description
command	str	Command to run.
force	bool	Force job to run even if runname already exists.
validatepath	bool	Validate the $PATH environment variable.
jobtype	JobType	Job type. Either batch or array.
queuetype	QueueType	Queue type. Either SGE or ...
runname	Path	Runname and run directory.
queue	str	Queue name.
freemem	ByteSizeGnu	Free memory requested.
maxmem	ByteSizeGnu	Maximum memory allowed.
procs	PositiveInt	Number of processors requested.
filesize	ByteSizeGnu	Maximum file size allowed.
concurrency	PositiveInt	Number of concurrent jobs.
ncommands	PositiveInt	Number of commands for array job. Auto-calculated.
tasklist	str	Tasklist to use. Deprecated.
priority	conint(ge=-1023, le=0	Job priority. Deprecated.
path	List[str]	List of paths to use. $PATH of current shell by default.
cmdtxt	Path	Command text file. Copied from input command.
cmdsh	Path	Command shell file. Submitted to the queuing system.
timestamp	str	Job timestamp. Also used for automatic runname generation.
cache	bool	Whether to cache job results. Not fully implemented yet.
localdrive	LocalType	Whether to use local drive.
mirrortype	MirrorType	Mirroring type when using local drive.
localprefix	Optional[Union[str, Path]]	Local drive prefix. Usually automatically generated.
copyresults	Optional[bool]	Whether to copy job results.
dryrun	Optional[bool]	Do not submit job, just generate the dir.
hold	Optional[str]	Hold for these jobs.
conda	bool	Enable conda in submit shell.
condaexe	Path	Use this conda exe for enable command.

Source code in hpcman/queue/submit.py

class Job(BaseModel):  # type: ignore
    """Job class based on Pydantic BaseModel.

    Raises:
        ValueError: When invalid inputs are provided.

    Attributes:
        command (str): Command to run.
        force (bool): Force job to run even if `runname` already exists.
        validatepath (bool): Validate the `$PATH` environment variable.
        jobtype (JobType): Job type. Either batch or array.
        queuetype (QueueType): Queue type. Either SGE or ...
        runname (Path): Runname and run directory.
        queue (str): Queue name.
        freemem (ByteSizeGnu): Free memory requested.
        maxmem (ByteSizeGnu): Maximum memory allowed.
        procs (PositiveInt): Number of processors requested.
        filesize (ByteSizeGnu): Maximum file size allowed.
        concurrency (PositiveInt): Number of concurrent jobs.
        ncommands (PositiveInt): Number of commands for array job. Auto-calculated.
        tasklist (str): Tasklist to use. Deprecated.
        priority (conint(ge=-1023, le=0)): Job priority. Deprecated.
        path (List[str]): List of paths to use. `$PATH` of current shell by default.
        cmdtxt (Path): Command text file. Copied from input command.
        cmdsh (Path): Command shell file. Submitted to the queuing system.
        timestamp (str): Job timestamp. Also used for automatic `runname` generation.
        cache (bool): Whether to cache job results. Not fully implemented yet.
        localdrive (LocalType): Whether to use local drive.
        mirrortype (MirrorType): Mirroring type when using local drive.
        localprefix (Optional[Union[str, Path]]): Local drive prefix. Usually automatically generated.
        copyresults (Optional[bool]): Whether to copy job results.
        dryrun (Optional[bool]): Do not submit job, just generate the dir.
        hold (Optional[str]): Hold for these jobs.
        conda (bool): Enable conda in submit shell.
        condaexe (Path): Use this conda exe for enable command.
    """

    command: str
    force: bool
    validatepath: bool
    jobtype: JobType
    queuetype: QueueType
    runname: Path
    queue: str
    freemem: Optional[ByteSizeGnu]
    maxmem: Optional[ByteSizeGnu]
    procs: PositiveInt
    filesize: Optional[ByteSizeGnu]
    concurrency: Optional[PositiveInt]
    ncommands: Optional[PositiveInt]
    tasklist: Optional[str]
    priority: Optional[conint(ge=-1023, le=0)]  # type: ignore
    path: List[str]
    cmdtxt: Optional[Path]
    cmdsh: Optional[Path]
    timestamp: str
    cache: bool
    localdrive: Optional[LocalType]
    mirrortype: MirrorType
    localprefix: Optional[Union[str, Path]]
    copyresults: Optional[bool]
    dryrun: Optional[bool]
    hold: Optional[str]
    conda: bool
    condaexe: Path

    # Need to add check for queuetype first
    @validator("runname", pre=True)  # type: ignore
    def check_for_past_runname(cls, v: Path, values: dict) -> Path:
        """Checks for existing `runname` directories.

        Args:
            v: Specified `runname` directory.
            values: Values of other options. Used to check for the 'force' option.

        Raises:
            ValueError: When `runname` already exists.
            ValueError: When `runname` is not a directory.

        Returns:
            `runname` directory that is ready for use.
        """
        while True:
            if v.exists():
                if v.is_dir():
                    if values["force"]:
                        response = JobPromptResponse.YES
                    else:
                        response = ConfirmJobDir.ask(f"Directory [red]{str(v)}[/red] already exists. Delete?")
                    if response is JobPromptResponse.YES:
                        rmtree(v)
                        break
                    elif response is JobPromptResponse.NEW:
                        v = Path(Prompt.ask("Please enter a new [green]runname[/green]"))
                        if not v.exists():
                            break
                    else:
                        raise ValueError(f"Delete rundir {v} before continuing.")
            elif v.is_file():
                raise ValueError(f"Specified rundir {v} is a file!")
            else:
                break
        return v

    # Need to add check for queuetype first
    @validator("runname", pre=True)  # type: ignore
    def check_for_valid_runname(cls, v: Path) -> Path:
        """Checks that `runname` is valid for SGE; SLURM specifics are not checked yet.

        Args:
            v: Specified `runname` directory.

        Raises:
            ValueError: When `runname` is not a valid SGE runname.

        Returns:
            Validated `runname`.
        """
        fntester = constr(regex=SGEJOBFILENAME)
        try:
            fntester.validate(str(v))  # pyright: ignore
        except StrRegexError as e:
            raise ValueError(
                f"Given rundir {v} is invalid. Must start with a letter and contain portable POSIX characters."
            ) from e
        return v

    @validator("path")  # type: ignore
    def check_for_valid_path(cls, v: List[str], values: dict) -> List[str]:
        """Checks that the specified `path` is valid.

        Args:
            v: Specified `path` directory.
            values: Other options. Used to check for the `validatepath` option.

        Raises:
            ValueError: If the `$SGE_ROOT` variable is not set.
            ValueError: If the `$SGE_ARCH` variable is not set.
            ValueError: If `/usr/bin`, `/local/cluster/sge/bin`, or the `bin` for the `$SGE_ARCH` variable is not in
                         the specified `path`.

        Returns:
            Validated `path`.
        """
        if not values["validatepath"]:
            return v
        standard_paths = ["/usr/bin"]
        if values["queuetype"] is QueueType.SGE:
            try:
                standard_paths.append(Path(environ["SGE_ROOT"], "bin").as_posix())
            except IndexError as e:
                raise ValueError(
                    "SGE_ROOT environment variable must be set to the path to the SGE root directory."
                ) from e
            try:
                standard_paths.append(Path(environ["SGE_ROOT"], "bin", environ["SGE_ARCH"]).as_posix())
            except IndexError as e:
                raise ValueError("SGE_ARCH environment variable must be set properly. Reload shell to continue.") from e
        elif values["queuetype"] is QueueType.SLURM:
            standard_paths.append(Path("/local", "cluster", "slurm", "bin").as_posix())
        for item in standard_paths:
            try:
                v.index(item)
            except ValueError as e:
                raise ValueError(f"Check your $PATH variable. Standard path {item} not found in $PATH.") from e
        return v

    def make_rundir(self, debug: bool = False) -> None:
        """Make rundir with potentially different settings per QueueType"""
        if self.queuetype in (QueueType.SGE, QueueType.SLURM):
            try:
                self.runname.mkdir(mode=0o755)
            except PermissionError:
                rprint(f"Unable to generate a directory in '{self.runname.parent.absolute().as_posix()}'")
                rprint("Please check your current directory and try again.")
                exit(1)

    def generate_commands_file(self, debug: bool = False) -> None:
        """Make commands.txt file"""
        if self.queuetype in (QueueType.SGE, QueueType.SLURM):
            self.cmdtxt = Path(self.runname, f"{self.runname}.commands")
            with open(self.cmdtxt, "wt", newline="\n") as cmdfile:
                if self.jobtype is JobType.BATCH:
                    cmdfile.write(
                        "\n".join(
                            [
                                "#!/usr/bin/env bash",
                                "# Added by hpcman queue submit\n",
                            ]
                        )
                    )
                    if self.conda:
                        cmdfile.write(
                            "\n".join(
                                [
                                    "# Enable `conda activate` and other conda commands",
                                    f'eval "$({self.condaexe} shell.bash hook)"\n',
                                ]
                            )
                        )
                cmdfile.write(f"{self.command}\n")

    def generate_submit_script(self, debug: bool = False) -> None:
        """Generates submit script depending on queuetype

        Uses all provided attributes to generate submit script. Will turn on request of e.g. processors or free/max
        memory or file size.

        Will produce submit scripts for each queue type.

        export NPROCS=--procs so that the processors requested can match the number given to the command.

        Args:
            debug: Used in development settings. Defaults to False.

        Raises:
            OSError: If the submit script cannot be created.
        """
        lines = []
        if self.queuetype is QueueType.SGE:
            lines = [
                "#!/usr/bin/env bash",
                "set -eo pipefail",
                "#",
                "# This file generated by hpcman queue submit",
                "#",
                "# Export all environment variables",
                "#$ -V",
                "#",
                "# Use current working directory",
                "#$ -cwd",
                "#",
                "# Use bash as the executing shell",
                "#$ -S /bin/bash",
                "#",
                "# Set the job name",
                f"#$ -N {self.runname}",
                "#",
                "# Set the queue name",
                f"#$ -q {self.queue}",
                "#",
                "# Output files for stdout and stderr",
                f"#$ -o {self.runname}",
                f"#$ -e {self.runname}",
                "#",
                f"export NPROCS={self.procs}",
            ]
            if self.procs > 1:
                lines.extend(
                    [
                        "# Request processors",
                        f"#$ -pe thread {self.procs}",
                        "#",
                    ]
                )
            if self.freemem or self.maxmem:
                lines.append("# Set the memory limit(s)")
                if self.freemem:
                    lines.append(f"#$ -l mem_free={self.freemem.human_readable(decimal=True, gnu=True)}")
                if self.maxmem:
                    lines.append(f"#$ -l h_vmem={self.maxmem.human_readable(decimal=True, gnu=True)}")
                lines.append("#")
            if self.filesize:
                lines.extend(
                    [
                        "# Set filesize limit",
                        f"#$ -l h_fsize={self.filesize.human_readable(decimal=True, gnu=True)}",
                        "#",
                    ]
                )
            if self.hold:
                lines.extend(["# Hold for these jobs", f"#$ -hold_jid {self.hold}", "#"])
            if self.jobtype is JobType.ARRAY:
                lines.extend(
                    [
                        "# Set task concurrency (max array jobs running simultaneously)",
                        f"#$ -tc {self.concurrency}",
                        "#",
                        "# Set array job range (1 to number of commands in cmd file)",
                        f"#$ -t 1-{self.ncommands}",
                        "#",
                    ]
                )
            lines.extend(
                [
                    "# Set PATH variable and submit directory",
                    f"export PATH={':'.join(self.path)}",
                    "#",
                    "submitdir=$(pwd)",
                ]
            )
            # Activate conda
            if self.localdrive:
                if self.localdrive is LocalType.PERTASK:
                    lines.extend(
                        [
                            "# Generate local dir per task",
                            f"mkdir -p {self.localprefix}",
                            f"workdir=$(mktemp -d -p {self.localprefix})",
                            "cd $workdir",
                        ]
                    )
                elif self.localdrive is LocalType.SHARED:
                    lines.extend(
                        [
                            "# Generate local dir shared by all tasks",
                            f"workdir={self.localprefix}",
                            "mkdir -p $workdir",
                            "cd $workdir",
                        ]
                    )
                if self.mirrortype is MirrorType.LINK:
                    lines.append("cp -ans $submitdir/* .")
                elif self.mirrortype is MirrorType.COPY:
                    lines.append("cp -an $submitdir/* .")
            # Print job information
            lines.extend(
                [
                    'echo "##hpcman.jobs={'
                    "'runid':'$JOB_ID',"
                    "'runname':'$JOB_NAME',"
                    "'host':'$(/bin/hostname -s)',"
                    "'wd':'$(pwd)',"
                    "'taskid':'$SGE_TASK_ID'"
                    "}" + '" >> /dev/stderr',
                    'echo "  Started on:\t $(/bin/hostname -s)"',
                    'echo "  Started at:\t $(/bin/date)"',
                ]
            )
            if self.jobtype is JobType.BATCH:
                if self.cmdtxt is not None:
                    lines.extend(
                        [
                            f'/usr/bin/time -f "{TIMEFMT}" \\',
                            f"\tbash $submitdir/{self.cmdtxt}",
                            rf"echo -e '\t{'Full Command:'.ljust(LPADDING)}bash {self.cmdtxt}'" " >> /dev/stderr",
                        ]
                    )
            elif self.jobtype is JobType.ARRAY:
                if self.cmdtxt is not None:
                    taskidsh = self.runname / f"{self.runname}.command.$SGE_TASK_ID"
                    lines.extend(
                        [
                            f'arraycmd=$(sed "$SGE_TASK_ID q;d" $submitdir/{self.cmdtxt})',
                            f'echo "#!/usr/bin/env bash" > $submitdir/{taskidsh}',
                        ]
                    )
                    if self.conda:
                        lines.extend(
                            [
                                f"echo '# Added by hpcman queue submit' >> $submitdir/{taskidsh}",
                                f"echo '# Enable `conda activate` and other conda commands' >> $submitdir/{taskidsh}",
                                "echo '" f'eval "$({self.condaexe} shell.bash hook)"' f"' >> $submitdir/{taskidsh}",
                                "#",
                            ]
                        )
                    lines.extend(
                        [
                            f"echo $arraycmd >> $submitdir/{taskidsh}",
                            f"chmod u+x $submitdir/{taskidsh}",
                            f'/usr/bin/time -f "{TIMEFMT}" \\',
                            f"\tbash $submitdir/{taskidsh}",
                            rf'echo -e "\t{"Full Command:".ljust(LPADDING)}{taskidsh}" >> /dev/stderr',
                        ]
                    )

            if self.localdrive and self.copyresults:
                lines.extend(
                    [
                        "if [ ! -z $workdir ]; then",
                        '\techo "copying results from $workdir to $submitdir and replacing with symlinks" '
                        ">> /dev/stderr",
                        "\trsync --ignore-existing --remove-source-files -av $workdir/* $submitdir/ >> /dev/stderr",
                        "\t# Generate symlinks, but send the error message about existing files to /dev/null and "
                        "return true.",
                        "\tcp -ans $submitdir/* . 2> /dev/null || true",
                        "fi",
                    ]
                )

            lines.append('echo "  Finished at:\t $(/bin/date)"')
        elif self.queuetype is QueueType.SLURM:
            lines = [
                "#!/usr/bin/env bash",
                "#",
                "# This file generated by hpcman queue submit",
                "#",
                "# Export all environment variables",
                "#SBATCH --export=ALL",
                "#",
                "# Use current working directory",
                f"#SBATCH -D '{self.runname.parent.absolute().as_posix()}'",
                "#",
                "# Set the job name",
                f"#SBATCH -J {self.runname}",
                "#",
                "# Output files for stdout and stderr",
                f"#SBATCH -o {self.runname}/{self.runname}.o%j",
                f"#SBATCH -e {self.runname}/{self.runname}.e%j",
                "#",
            ]
            if self.queue != "*":
                lines.extend(
                    [
                        "# Set the queue name",
                        f"#SBATCH -p {self.queue}",
                        "#",
                    ]
                )
            if self.procs > 1:
                lines.extend(
                    [
                        "# Request processors",
                        f"#SBATCH -c {self.procs}",
                        "#",
                    ]
                )
            if self.freemem:
                lines.append("# Set the memory limit(s)")
                if self.freemem:
                    lines.append(f"#SBATCH --mem={self.freemem.human_readable(decimal=True, gnu=True, slurm=True)}")
                # if self.maxmem:
                #     lines.append(f"#$ -l h_vmem={self.maxmem.human_readable(decimal=True, gnu=True)}")
                lines.append("#")
            # if self.filesize:
            #     lines.extend(
            #         [
            #             "# Set filesize limit",
            #             f"#$ -l h_fsize={self.filesize.human_readable(decimal=True, gnu=True)}",
            #             "#",
            #         ]
            #     )
            if self.hold:
                lines.extend(["# Hold for these jobs", f"#SBATCH -d afterok:{':'.join(self.hold)}", "#"])
            if self.jobtype is JobType.ARRAY:
                lines.extend(
                    [
                        "# Set array job range (1 to number of commands in cmd file)",
                        "# Set task concurrency (max array jobs running simultaneously)",
                        f"#SBATCH -a 1-{self.ncommands}%{self.concurrency}",
                        "#",
                    ]
                )
            lines.extend(
                [
                    "set -eo pipefail",
                    f"export NPROCS={self.procs}",
                    "# Set PATH variable and submit directory",
                    f"export PATH={':'.join(self.path)}",
                    "#",
                    "submitdir=$(pwd)",
                ]
            )
            # Activate conda
            if self.localdrive:
                if self.localdrive is LocalType.PERTASK:
                    lines.extend(
                        [
                            "# Generate local dir per task",
                            f"mkdir -p {self.localprefix}",
                            f"workdir=$(mktemp -d -p {self.localprefix})",
                            "cd $workdir",
                        ]
                    )
                elif self.localdrive is LocalType.SHARED:
                    lines.extend(
                        [
                            "# Generate local dir shared by all tasks",
                            f"workdir={self.localprefix}",
                            "mkdir -p $workdir",
                            "cd $workdir",
                        ]
                    )
                if self.mirrortype is MirrorType.LINK:
                    lines.append("cp -ans $submitdir/* .")
                elif self.mirrortype is MirrorType.COPY:
                    lines.append("cp -an $submitdir/* .")
            # Print job information
            lines.extend(
                [
                    'echo "##hpcman.jobs={'
                    "'runid':'$SLURM_JOB_ID',"
                    "'runname':'$SLURM_JOB_NAME',"
                    "'host':'$(/bin/hostname -s)',"
                    "'wd':'$(pwd)',"
                    "'taskid':'$SLURM_ARRAY_TASK_ID'"
                    "}" + '" >> /dev/stderr',
                    'echo "  Started on:\t $(/bin/hostname -s)"',
                    'echo "  Started at:\t $(/bin/date)"',
                ]
            )
            if self.jobtype is JobType.BATCH:
                if self.cmdtxt is not None:
                    lines.extend(
                        [
                            f'/usr/bin/time -f "{TIMEFMT}" \\',
                            f"\tbash $submitdir/{self.cmdtxt}",
                            rf"echo -e '\t{'Full Command:'.ljust(LPADDING)}bash {self.cmdtxt}'" " >> /dev/stderr",
                        ]
                    )
            elif self.jobtype is JobType.ARRAY:
                if self.cmdtxt is not None:
                    taskidsh = self.runname / f"{self.runname}.command.$SLURM_ARRAY_TASK_ID"
                    lines.extend(
                        [
                            f'arraycmd=$(sed "$SLURM_ARRAY_TASK_ID q;d" $submitdir/{self.cmdtxt})',
                            f'echo "#!/usr/bin/env bash" > $submitdir/{taskidsh}',
                        ]
                    )
                    if self.conda:
                        lines.extend(
                            [
                                f"echo '# Added by hpcman queue submit' >> $submitdir/{taskidsh}",
                                f"echo '# Enable `conda activate` and other conda commands' >> $submitdir/{taskidsh}",
                                "echo '" f'eval "$({self.condaexe} shell.bash hook)"' f"' >> $submitdir/{taskidsh}",
                                "#",
                            ]
                        )
                    lines.extend(
                        [
                            f"echo $arraycmd >> $submitdir/{taskidsh}",
                            f"chmod u+x $submitdir/{taskidsh}",
                            f'/usr/bin/time -f "{TIMEFMT}" \\',
                            f"\tbash $submitdir/{taskidsh}",
                            rf'echo -e "\t{"Full Command:".ljust(LPADDING)}{taskidsh}" >> /dev/stderr',
                        ]
                    )
            lines.append('echo "  Finished at:\t $(/bin/date)"')

        self.cmdsh = Path(self.runname, f"{self.runname}.sh")
        try:
            self.cmdsh.write_text("\n".join(lines) + "\n")
        except OSError as e:
            print("Unable to generate the submit script. Check folder/permissions and try again.")
            raise e

    def submit(self, debug: bool = False) -> None:
        """Submits the generated submit script to the appropriate queue

        Prints a success message and logs data to the `.hpcman.jobnums` file, as well at the `~/hpcman/cache` directory.

        Args:
            debug: Used in development settings. Defaults to False.

        Raises:
            CalledProcessError: When the job cannot be submitted.
        """
        # Mostly to appease pylance type checking
        if self.cmdsh is None:
            raise ValueError(f"No submit script found for {self}")
        if self.queuetype is QueueType.SGE:
            cmd = []
            cmd = ["qsub", "-terse", self.cmdsh.as_posix()]
            if self.dryrun:
                rprint(
                    "==! :stop_sign: [red]--dry-run[/red] specified. Job has [bold red]NOT[/bold red] been "
                    "submitted !=="
                )
                rprint(f"To submit the job, run: [green]qsub {self.cmdsh.as_posix()}[/green]")
                rprint("Job info will not be logged to the [green].hpcman.jobnums[/green] file.")
                exit(0)
        if self.queuetype is QueueType.SLURM:
            cmd = []
            cmd = ["sbatch", "--parsable", self.cmdsh.as_posix()]
            if self.dryrun:
                rprint(
                    "==! :stop_sign: [red]--dry-run[/red] specified. Job has [bold red]NOT[/bold red] been "
                    "submitted !=="
                )
                rprint(f"To submit the job, run: [green]sbatch {self.cmdsh.as_posix()}[/green]")
                rprint("Job info will not be logged to the [green].hpcman.jobnums[/green] file.")
                exit(0)
        if not debug:
            try:
                res = subprocess.run(cmd, check=True, capture_output=True, text=True)
            except CalledProcessError as e:
                rprint(f"Removing run directory {self.runname}")
                rmtree(self.runname)
                if "unknown queue" in e.stderr:
                    rprint(f"Please choose a queue from this list and try again: {get_queue_list()}")
                    exit(1)
                else:
                    rprint(f"There was a problem submitting the job: '{e.stderr}'")
                    raise e
            runid = res.stdout.strip()
        else:
            runid = str(random.randint(1, 999999))
        rprint(
            f":tada: [green]Successfully[/green] submitted job {runid} [green]{self.runname}[/green] to queue "
            f"[cyan]{self.queue}[/cyan], logging job number, timestamp, and runname to "
            "[green].hpcman.jobnums[/green]"
        )
        write_log(
            logtype=LogType.FILE,
            runid=runid,
            timestamp=self.timestamp,
            runname=self.runname,
            target=Path(".hpcman.jobnums"),
            queue=None,
        )
        if self.cache:
            write_log(
                logtype=LogType.DB,
                runid=runid,
                timestamp=self.timestamp,
                runname=self.runname,
                queue=self.queue,
                target=Path("~/hpcman/cache/jobs.db"),
            )

    def __str__(self) -> str:
        message = (
            "Job parameters:\n"
            f"\trunname     -> {self.runname}\n"
            f"\tqueue       -> {self.queue}\n"
            f"\tprocessors  -> {self.procs}\n"
            f"\tjobtype     -> {self.jobtype.value}\n"
            f"\tqueuetype   -> {self.queuetype.value}\n"
            f"\tfreemem     -> {self.freemem.human_readable(gnu=True, decimal=True) if self.freemem else None}\n"
            f"\tmaxmem      -> {self.maxmem.human_readable(gnu=True, decimal=True) if self.maxmem else None}\n"
            f"\tfilesize    -> {self.filesize.human_readable(gnu=True, decimal=True) if self.filesize else None}\n"
            f"\tconcurrency -> {self.concurrency if self.jobtype is JobType.ARRAY else None}\n"
            "Command(s):\n"
            f"\t{self.command}\n"
        )
        return message

check_for_past_runname(v, values)

Checks for existing runname directories.

Parameters:

Name	Type	Description	Default
v	Path	Specified runname directory.	required
values	dict	Values of other options. Used to check for the 'force' option.	required

Raises:

Type	Description
ValueError	When runname already exists.
ValueError	When runname is not a directory.

Returns:

Type	Description
Path	runname directory that is ready for use.

Source code in hpcman/queue/submit.py

@validator("runname", pre=True)  # type: ignore
def check_for_past_runname(cls, v: Path, values: dict) -> Path:
    """Checks for existing `runname` directories.

    Args:
        v: Specified `runname` directory.
        values: Values of other options. Used to check for the 'force' option.

    Raises:
        ValueError: When `runname` already exists.
        ValueError: When `runname` is not a directory.

    Returns:
        `runname` directory that is ready for use.
    """
    while True:
        if v.exists():
            if v.is_dir():
                if values["force"]:
                    response = JobPromptResponse.YES
                else:
                    response = ConfirmJobDir.ask(f"Directory [red]{str(v)}[/red] already exists. Delete?")
                if response is JobPromptResponse.YES:
                    rmtree(v)
                    break
                elif response is JobPromptResponse.NEW:
                    v = Path(Prompt.ask("Please enter a new [green]runname[/green]"))
                    if not v.exists():
                        break
                else:
                    raise ValueError(f"Delete rundir {v} before continuing.")
        elif v.is_file():
            raise ValueError(f"Specified rundir {v} is a file!")
        else:
            break
    return v

check_for_valid_path(v, values)

Checks that the specified path is valid.

Parameters:

Name	Type	Description	Default
v	List[str]	Specified path directory.	required
values	dict	Other options. Used to check for the validatepath option.	required

Raises:

Type	Description
ValueError	If the $SGE_ROOT variable is not set.
ValueError	If the $SGE_ARCH variable is not set.
ValueError	If /usr/bin, /local/cluster/sge/bin, or the bin for the $SGE_ARCH variable is not in the specified path.

Returns:

Type	Description
List[str]	Validated path.

Source code in hpcman/queue/submit.py

@validator("path")  # type: ignore
def check_for_valid_path(cls, v: List[str], values: dict) -> List[str]:
    """Checks that the specified `path` is valid.

    Args:
        v: Specified `path` directory.
        values: Other options. Used to check for the `validatepath` option.

    Raises:
        ValueError: If the `$SGE_ROOT` variable is not set.
        ValueError: If the `$SGE_ARCH` variable is not set.
        ValueError: If `/usr/bin`, `/local/cluster/sge/bin`, or the `bin` for the `$SGE_ARCH` variable is not in
                     the specified `path`.

    Returns:
        Validated `path`.
    """
    if not values["validatepath"]:
        return v
    standard_paths = ["/usr/bin"]
    if values["queuetype"] is QueueType.SGE:
        try:
            standard_paths.append(Path(environ["SGE_ROOT"], "bin").as_posix())
        except IndexError as e:
            raise ValueError(
                "SGE_ROOT environment variable must be set to the path to the SGE root directory."
            ) from e
        try:
            standard_paths.append(Path(environ["SGE_ROOT"], "bin", environ["SGE_ARCH"]).as_posix())
        except IndexError as e:
            raise ValueError("SGE_ARCH environment variable must be set properly. Reload shell to continue.") from e
    elif values["queuetype"] is QueueType.SLURM:
        standard_paths.append(Path("/local", "cluster", "slurm", "bin").as_posix())
    for item in standard_paths:
        try:
            v.index(item)
        except ValueError as e:
            raise ValueError(f"Check your $PATH variable. Standard path {item} not found in $PATH.") from e
    return v

check_for_valid_runname(v)

Checks that runname is valid for SGE; SLURM specifics are not checked yet.

Parameters:

Name	Type	Description	Default
v	Path	Specified runname directory.	required

Raises:

Type	Description
ValueError	When runname is not a valid SGE runname.

Returns:

Type	Description
Path	Validated runname.

Source code in hpcman/queue/submit.py

@validator("runname", pre=True)  # type: ignore
def check_for_valid_runname(cls, v: Path) -> Path:
    """Checks that `runname` is valid for SGE; SLURM specifics are not checked yet.

    Args:
        v: Specified `runname` directory.

    Raises:
        ValueError: When `runname` is not a valid SGE runname.

    Returns:
        Validated `runname`.
    """
    fntester = constr(regex=SGEJOBFILENAME)
    try:
        fntester.validate(str(v))  # pyright: ignore
    except StrRegexError as e:
        raise ValueError(
            f"Given rundir {v} is invalid. Must start with a letter and contain portable POSIX characters."
        ) from e
    return v

generate_commands_file(debug=False)

Make commands.txt file

Source code in hpcman/queue/submit.py

def generate_commands_file(self, debug: bool = False) -> None:
    """Make commands.txt file"""
    if self.queuetype in (QueueType.SGE, QueueType.SLURM):
        self.cmdtxt = Path(self.runname, f"{self.runname}.commands")
        with open(self.cmdtxt, "wt", newline="\n") as cmdfile:
            if self.jobtype is JobType.BATCH:
                cmdfile.write(
                    "\n".join(
                        [
                            "#!/usr/bin/env bash",
                            "# Added by hpcman queue submit\n",
                        ]
                    )
                )
                if self.conda:
                    cmdfile.write(
                        "\n".join(
                            [
                                "# Enable `conda activate` and other conda commands",
                                f'eval "$({self.condaexe} shell.bash hook)"\n',
                            ]
                        )
                    )
            cmdfile.write(f"{self.command}\n")

generate_submit_script(debug=False)

Generates submit script depending on queuetype

Uses all provided attributes to generate submit script. Will turn on request of e.g. processors or free/max memory or file size.

Will produce submit scripts for each queue type.

export NPROCS=--procs so that the processors requested can match the number given to the command.

Parameters:

Name	Type	Description	Default
debug	bool	Used in development settings. Defaults to False.	False

Raises:

Type	Description
OSError	If the submit script cannot be created.

Source code in hpcman/queue/submit.py

def generate_submit_script(self, debug: bool = False) -> None:
    """Generates submit script depending on queuetype

    Uses all provided attributes to generate submit script. Will turn on request of e.g. processors or free/max
    memory or file size.

    Will produce submit scripts for each queue type.

    export NPROCS=--procs so that the processors requested can match the number given to the command.

    Args:
        debug: Used in development settings. Defaults to False.

    Raises:
        OSError: If the submit script cannot be created.
    """
    lines = []
    if self.queuetype is QueueType.SGE:
        lines = [
            "#!/usr/bin/env bash",
            "set -eo pipefail",
            "#",
            "# This file generated by hpcman queue submit",
            "#",
            "# Export all environment variables",
            "#$ -V",
            "#",
            "# Use current working directory",
            "#$ -cwd",
            "#",
            "# Use bash as the executing shell",
            "#$ -S /bin/bash",
            "#",
            "# Set the job name",
            f"#$ -N {self.runname}",
            "#",
            "# Set the queue name",
            f"#$ -q {self.queue}",
            "#",
            "# Output files for stdout and stderr",
            f"#$ -o {self.runname}",
            f"#$ -e {self.runname}",
            "#",
            f"export NPROCS={self.procs}",
        ]
        if self.procs > 1:
            lines.extend(
                [
                    "# Request processors",
                    f"#$ -pe thread {self.procs}",
                    "#",
                ]
            )
        if self.freemem or self.maxmem:
            lines.append("# Set the memory limit(s)")
            if self.freemem:
                lines.append(f"#$ -l mem_free={self.freemem.human_readable(decimal=True, gnu=True)}")
            if self.maxmem:
                lines.append(f"#$ -l h_vmem={self.maxmem.human_readable(decimal=True, gnu=True)}")
            lines.append("#")
        if self.filesize:
            lines.extend(
                [
                    "# Set filesize limit",
                    f"#$ -l h_fsize={self.filesize.human_readable(decimal=True, gnu=True)}",
                    "#",
                ]
            )
        if self.hold:
            lines.extend(["# Hold for these jobs", f"#$ -hold_jid {self.hold}", "#"])
        if self.jobtype is JobType.ARRAY:
            lines.extend(
                [
                    "# Set task concurrency (max array jobs running simultaneously)",
                    f"#$ -tc {self.concurrency}",
                    "#",
                    "# Set array job range (1 to number of commands in cmd file)",
                    f"#$ -t 1-{self.ncommands}",
                    "#",
                ]
            )
        lines.extend(
            [
                "# Set PATH variable and submit directory",
                f"export PATH={':'.join(self.path)}",
                "#",
                "submitdir=$(pwd)",
            ]
        )
        # Activate conda
        if self.localdrive:
            if self.localdrive is LocalType.PERTASK:
                lines.extend(
                    [
                        "# Generate local dir per task",
                        f"mkdir -p {self.localprefix}",
                        f"workdir=$(mktemp -d -p {self.localprefix})",
                        "cd $workdir",
                    ]
                )
            elif self.localdrive is LocalType.SHARED:
                lines.extend(
                    [
                        "# Generate local dir shared by all tasks",
                        f"workdir={self.localprefix}",
                        "mkdir -p $workdir",
                        "cd $workdir",
                    ]
                )
            if self.mirrortype is MirrorType.LINK:
                lines.append("cp -ans $submitdir/* .")
            elif self.mirrortype is MirrorType.COPY:
                lines.append("cp -an $submitdir/* .")
        # Print job information
        lines.extend(
            [
                'echo "##hpcman.jobs={'
                "'runid':'$JOB_ID',"
                "'runname':'$JOB_NAME',"
                "'host':'$(/bin/hostname -s)',"
                "'wd':'$(pwd)',"
                "'taskid':'$SGE_TASK_ID'"
                "}" + '" >> /dev/stderr',
                'echo "  Started on:\t $(/bin/hostname -s)"',
                'echo "  Started at:\t $(/bin/date)"',
            ]
        )
        if self.jobtype is JobType.BATCH:
            if self.cmdtxt is not None:
                lines.extend(
                    [
                        f'/usr/bin/time -f "{TIMEFMT}" \\',
                        f"\tbash $submitdir/{self.cmdtxt}",
                        rf"echo -e '\t{'Full Command:'.ljust(LPADDING)}bash {self.cmdtxt}'" " >> /dev/stderr",
                    ]
                )
        elif self.jobtype is JobType.ARRAY:
            if self.cmdtxt is not None:
                taskidsh = self.runname / f"{self.runname}.command.$SGE_TASK_ID"
                lines.extend(
                    [
                        f'arraycmd=$(sed "$SGE_TASK_ID q;d" $submitdir/{self.cmdtxt})',
                        f'echo "#!/usr/bin/env bash" > $submitdir/{taskidsh}',
                    ]
                )
                if self.conda:
                    lines.extend(
                        [
                            f"echo '# Added by hpcman queue submit' >> $submitdir/{taskidsh}",
                            f"echo '# Enable `conda activate` and other conda commands' >> $submitdir/{taskidsh}",
                            "echo '" f'eval "$({self.condaexe} shell.bash hook)"' f"' >> $submitdir/{taskidsh}",
                            "#",
                        ]
                    )
                lines.extend(
                    [
                        f"echo $arraycmd >> $submitdir/{taskidsh}",
                        f"chmod u+x $submitdir/{taskidsh}",
                        f'/usr/bin/time -f "{TIMEFMT}" \\',
                        f"\tbash $submitdir/{taskidsh}",
                        rf'echo -e "\t{"Full Command:".ljust(LPADDING)}{taskidsh}" >> /dev/stderr',
                    ]
                )

        if self.localdrive and self.copyresults:
            lines.extend(
                [
                    "if [ ! -z $workdir ]; then",
                    '\techo "copying results from $workdir to $submitdir and replacing with symlinks" '
                    ">> /dev/stderr",
                    "\trsync --ignore-existing --remove-source-files -av $workdir/* $submitdir/ >> /dev/stderr",
                    "\t# Generate symlinks, but send the error message about existing files to /dev/null and "
                    "return true.",
                    "\tcp -ans $submitdir/* . 2> /dev/null || true",
                    "fi",
                ]
            )

        lines.append('echo "  Finished at:\t $(/bin/date)"')
    elif self.queuetype is QueueType.SLURM:
        lines = [
            "#!/usr/bin/env bash",
            "#",
            "# This file generated by hpcman queue submit",
            "#",
            "# Export all environment variables",
            "#SBATCH --export=ALL",
            "#",
            "# Use current working directory",
            f"#SBATCH -D '{self.runname.parent.absolute().as_posix()}'",
            "#",
            "# Set the job name",
            f"#SBATCH -J {self.runname}",
            "#",
            "# Output files for stdout and stderr",
            f"#SBATCH -o {self.runname}/{self.runname}.o%j",
            f"#SBATCH -e {self.runname}/{self.runname}.e%j",
            "#",
        ]
        if self.queue != "*":
            lines.extend(
                [
                    "# Set the queue name",
                    f"#SBATCH -p {self.queue}",
                    "#",
                ]
            )
        if self.procs > 1:
            lines.extend(
                [
                    "# Request processors",
                    f"#SBATCH -c {self.procs}",
                    "#",
                ]
            )
        if self.freemem:
            lines.append("# Set the memory limit(s)")
            if self.freemem:
                lines.append(f"#SBATCH --mem={self.freemem.human_readable(decimal=True, gnu=True, slurm=True)}")
            # if self.maxmem:
            #     lines.append(f"#$ -l h_vmem={self.maxmem.human_readable(decimal=True, gnu=True)}")
            lines.append("#")
        # if self.filesize:
        #     lines.extend(
        #         [
        #             "# Set filesize limit",
        #             f"#$ -l h_fsize={self.filesize.human_readable(decimal=True, gnu=True)}",
        #             "#",
        #         ]
        #     )
        if self.hold:
            lines.extend(["# Hold for these jobs", f"#SBATCH -d afterok:{':'.join(self.hold)}", "#"])
        if self.jobtype is JobType.ARRAY:
            lines.extend(
                [
                    "# Set array job range (1 to number of commands in cmd file)",
                    "# Set task concurrency (max array jobs running simultaneously)",
                    f"#SBATCH -a 1-{self.ncommands}%{self.concurrency}",
                    "#",
                ]
            )
        lines.extend(
            [
                "set -eo pipefail",
                f"export NPROCS={self.procs}",
                "# Set PATH variable and submit directory",
                f"export PATH={':'.join(self.path)}",
                "#",
                "submitdir=$(pwd)",
            ]
        )
        # Activate conda
        if self.localdrive:
            if self.localdrive is LocalType.PERTASK:
                lines.extend(
                    [
                        "# Generate local dir per task",
                        f"mkdir -p {self.localprefix}",
                        f"workdir=$(mktemp -d -p {self.localprefix})",
                        "cd $workdir",
                    ]
                )
            elif self.localdrive is LocalType.SHARED:
                lines.extend(
                    [
                        "# Generate local dir shared by all tasks",
                        f"workdir={self.localprefix}",
                        "mkdir -p $workdir",
                        "cd $workdir",
                    ]
                )
            if self.mirrortype is MirrorType.LINK:
                lines.append("cp -ans $submitdir/* .")
            elif self.mirrortype is MirrorType.COPY:
                lines.append("cp -an $submitdir/* .")
        # Print job information
        lines.extend(
            [
                'echo "##hpcman.jobs={'
                "'runid':'$SLURM_JOB_ID',"
                "'runname':'$SLURM_JOB_NAME',"
                "'host':'$(/bin/hostname -s)',"
                "'wd':'$(pwd)',"
                "'taskid':'$SLURM_ARRAY_TASK_ID'"
                "}" + '" >> /dev/stderr',
                'echo "  Started on:\t $(/bin/hostname -s)"',
                'echo "  Started at:\t $(/bin/date)"',
            ]
        )
        if self.jobtype is JobType.BATCH:
            if self.cmdtxt is not None:
                lines.extend(
                    [
                        f'/usr/bin/time -f "{TIMEFMT}" \\',
                        f"\tbash $submitdir/{self.cmdtxt}",
                        rf"echo -e '\t{'Full Command:'.ljust(LPADDING)}bash {self.cmdtxt}'" " >> /dev/stderr",
                    ]
                )
        elif self.jobtype is JobType.ARRAY:
            if self.cmdtxt is not None:
                taskidsh = self.runname / f"{self.runname}.command.$SLURM_ARRAY_TASK_ID"
                lines.extend(
                    [
                        f'arraycmd=$(sed "$SLURM_ARRAY_TASK_ID q;d" $submitdir/{self.cmdtxt})',
                        f'echo "#!/usr/bin/env bash" > $submitdir/{taskidsh}',
                    ]
                )
                if self.conda:
                    lines.extend(
                        [
                            f"echo '# Added by hpcman queue submit' >> $submitdir/{taskidsh}",
                            f"echo '# Enable `conda activate` and other conda commands' >> $submitdir/{taskidsh}",
                            "echo '" f'eval "$({self.condaexe} shell.bash hook)"' f"' >> $submitdir/{taskidsh}",
                            "#",
                        ]
                    )
                lines.extend(
                    [
                        f"echo $arraycmd >> $submitdir/{taskidsh}",
                        f"chmod u+x $submitdir/{taskidsh}",
                        f'/usr/bin/time -f "{TIMEFMT}" \\',
                        f"\tbash $submitdir/{taskidsh}",
                        rf'echo -e "\t{"Full Command:".ljust(LPADDING)}{taskidsh}" >> /dev/stderr',
                    ]
                )
        lines.append('echo "  Finished at:\t $(/bin/date)"')

    self.cmdsh = Path(self.runname, f"{self.runname}.sh")
    try:
        self.cmdsh.write_text("\n".join(lines) + "\n")
    except OSError as e:
        print("Unable to generate the submit script. Check folder/permissions and try again.")
        raise e

make_rundir(debug=False)

Make rundir with potentially different settings per QueueType

Source code in hpcman/queue/submit.py

def make_rundir(self, debug: bool = False) -> None:
    """Make rundir with potentially different settings per QueueType"""
    if self.queuetype in (QueueType.SGE, QueueType.SLURM):
        try:
            self.runname.mkdir(mode=0o755)
        except PermissionError:
            rprint(f"Unable to generate a directory in '{self.runname.parent.absolute().as_posix()}'")
            rprint("Please check your current directory and try again.")
            exit(1)

submit(debug=False)

Submits the generated submit script to the appropriate queue

Prints a success message and logs data to the .hpcman.jobnums file, as well at the ~/hpcman/cache directory.

Parameters:

Name	Type	Description	Default
debug	bool	Used in development settings. Defaults to False.	False

Raises:

Type	Description
CalledProcessError	When the job cannot be submitted.

Source code in hpcman/queue/submit.py

def submit(self, debug: bool = False) -> None:
    """Submits the generated submit script to the appropriate queue

    Prints a success message and logs data to the `.hpcman.jobnums` file, as well at the `~/hpcman/cache` directory.

    Args:
        debug: Used in development settings. Defaults to False.

    Raises:
        CalledProcessError: When the job cannot be submitted.
    """
    # Mostly to appease pylance type checking
    if self.cmdsh is None:
        raise ValueError(f"No submit script found for {self}")
    if self.queuetype is QueueType.SGE:
        cmd = []
        cmd = ["qsub", "-terse", self.cmdsh.as_posix()]
        if self.dryrun:
            rprint(
                "==! :stop_sign: [red]--dry-run[/red] specified. Job has [bold red]NOT[/bold red] been "
                "submitted !=="
            )
            rprint(f"To submit the job, run: [green]qsub {self.cmdsh.as_posix()}[/green]")
            rprint("Job info will not be logged to the [green].hpcman.jobnums[/green] file.")
            exit(0)
    if self.queuetype is QueueType.SLURM:
        cmd = []
        cmd = ["sbatch", "--parsable", self.cmdsh.as_posix()]
        if self.dryrun:
            rprint(
                "==! :stop_sign: [red]--dry-run[/red] specified. Job has [bold red]NOT[/bold red] been "
                "submitted !=="
            )
            rprint(f"To submit the job, run: [green]sbatch {self.cmdsh.as_posix()}[/green]")
            rprint("Job info will not be logged to the [green].hpcman.jobnums[/green] file.")
            exit(0)
    if not debug:
        try:
            res = subprocess.run(cmd, check=True, capture_output=True, text=True)
        except CalledProcessError as e:
            rprint(f"Removing run directory {self.runname}")
            rmtree(self.runname)
            if "unknown queue" in e.stderr:
                rprint(f"Please choose a queue from this list and try again: {get_queue_list()}")
                exit(1)
            else:
                rprint(f"There was a problem submitting the job: '{e.stderr}'")
                raise e
        runid = res.stdout.strip()
    else:
        runid = str(random.randint(1, 999999))
    rprint(
        f":tada: [green]Successfully[/green] submitted job {runid} [green]{self.runname}[/green] to queue "
        f"[cyan]{self.queue}[/cyan], logging job number, timestamp, and runname to "
        "[green].hpcman.jobnums[/green]"
    )
    write_log(
        logtype=LogType.FILE,
        runid=runid,
        timestamp=self.timestamp,
        runname=self.runname,
        target=Path(".hpcman.jobnums"),
        queue=None,
    )
    if self.cache:
        write_log(
            logtype=LogType.DB,
            runid=runid,
            timestamp=self.timestamp,
            runname=self.runname,
            queue=self.queue,
            target=Path("~/hpcman/cache/jobs.db"),
        )

check_if_submit_host(queuetype, debug)

Checks if the current host is a submit host.

Uses the qconf program to determine if the current host is a submit host.

Need to implement a similar feature for SLURM.

Parameters:

Name	Type	Description	Default
queuetype	QueueType	What queue type to check.	required
debug	bool	Used in development settings. SubmitHosts can be added in the enum.py file for debugging.	required

Raises:

Type	Description
CalledProcessError	When there is a problem with the qconf program.
NotASubmitHost	When the current host is not a submit host.
NotImplementedError	When the queuetype is not supported.

Returns:

Type	Description
str	hostname for current node.

Source code in hpcman/queue/submit.py

def check_if_submit_host(queuetype: QueueType, debug: bool) -> str:
    """Checks if the current host is a submit host.

    Uses the `qconf` program to determine if the current host is a submit host.

    Need to implement a similar feature for SLURM.

    Args:
        queuetype: What queue type to check.
        debug: Used in development settings. SubmitHosts can be added in the `enum.py` file for debugging.

    Raises:
        CalledProcessError: When there is a problem with the `qconf` program.
        NotASubmitHost: When the current host is not a submit host.
        NotImplementedError: When the queuetype is not supported.

    Returns:
        `hostname` for current node.
    """
    hostname = platform.node()
    if debug:
        submit_hosts: List[str] = [el.value for el in SubmitHosts]
    else:
        if queuetype is QueueType.SGE:
            try:
                submit_hosts = subprocess.run(["qconf", "-ss"], capture_output=True, text=True).stdout.split()
            except CalledProcessError as e:
                print(f"There was a problem checking the submit host: {e.stderr}")
                raise e
            if hostname not in submit_hosts and hostname not in [el.value for el in SLURMHosts]:
                raise NotASubmitHost(f"{hostname} is not a submit host. Try again on shell.cqls.oregonstate.edu.")
        elif queuetype is QueueType.SLURM:
            submit_hosts = [el.value for el in SLURMHosts]
            if hostname not in submit_hosts:
                raise NotASubmitHost(f"{hostname} is not a submit host. Try again on shell-hpc.cqls.oregonstate.edu.")
        else:
            raise NotImplementedError(f"Cannot find submit host for {queuetype}")
    return hostname

generate_runname(cmd, timestamp)

Generate a unique runname for a given command and timestamp

Parameters:

Name	Type	Description	Default
cmd	str	Specified command to extract a portion from	required
timestamp	str	Calculated timestamp to use in the runname	required

Returns:

Type	Description
str	Generated runname.

Source code in hpcman/queue/submit.py

def generate_runname(cmd: str, timestamp: str) -> str:
    """Generate a unique runname for a given command and timestamp

    Args:
        cmd: Specified command to extract a portion from
        timestamp: Calculated timestamp to use in the runname

    Returns:
        Generated runname.
    """
    cmd = Path(re.split(r"\s+", cmd, 1)[0]).name
    cmd = re.sub(r"[^a-zA-Z0-9]", "", cmd)
    return f"j{timestamp}_{cmd}_etal"

get_holds()

Gets keys from the .hpcman.jobnums file.

The keys are presently the jobids. Other values are ignored.

Source code in hpcman/queue/submit.py

def get_holds() -> List[str]:
    """Gets keys from the .hpcman.jobnums file.

    The keys are presently the jobids. Other values are ignored.
    """
    import json

    jids = []
    jobnumf = Path(".hpcman.jobnums")
    with jobnumf.open("rt") as f:
        for line in f:
            jids.extend(json.loads(line).keys())
    return [x.split(".")[0] for x in jids]

submit_job(job, debug)

Submit job depending on queuetype and other provided parameters.

Methods are run from the Job instance.

Parameters:

Name	Type	Description	Default
job	Job	Job instance to submit.	required
debug	bool	Used in development settings.	required

Source code in hpcman/queue/submit.py

def submit_job(job: Job, debug: bool) -> None:
    """Submit job depending on queuetype and other provided parameters.

    Methods are run from the `Job` instance.

    Args:
        job: `Job` instance to submit.
        debug: Used in development settings.
    """
    check_if_submit_host(job.queuetype, debug)
    job.make_rundir()
    job.generate_commands_file(debug)
    job.generate_submit_script(debug)
    job.submit(debug)