Create Notebooks but not run them

One should check how-to-run-notebook-on-cluster.ipynb for executing the notebooks on a cluster. For here, we only generate evaluation notebooks if they do not exist. Thus, you can effectively force a rerun, or update the notebook to the latest version, by removing the old notebooks. You can also choose to remove the JSON data or not, depending on your needs.

Tips

When you first run some notebook from scratch, try to use a small number of workers (e.g., slurm_maximum_jobs=30). This is because normally at the beginning of the evaluation, the jobs are highly sequential. At this stage, the submitter and the executor are still trying to figure out how fast each shot runs and how many shots we need to gather. They don’t want to submit too much jobs that it takes forever to execute. After it gathers enough data, e.g., has gone to the next (higher precision) submitter, then you can increase the number of workers because those jobs are highly parallelizable.

%load_ext autoreload
%autoreload 2

import os, sys, time
import papermill as pm
from dotmap import DotMap as dmap
from slugify import slugify
from qec_lego_bench.hpc.cluster_util import tmux_list_script
import nbstripout  # please install, and we will call it from command line
import subprocess

template_path = 'template.ipynb'

inspected = pm.inspect_notebook(template_path)
print("available keys")
for key, value in inspected.items():
    v = dmap(value)
    print(f"    - {key}: {v.inferred_type_name} = {v.default}")

# remove all the output from the notebook
def remove_notebook_output(nb_path: str):
    subprocess.Popen(f"nbstripout {nb_path}", shell=True)

available keys
    - decoder_eval: str = '"mwpm"'
    - code_eval: str = 'f"rsc(d@{c.d},p@{p})"'
    - noise_eval: str = '"none"'
    - config_vec_eval: str = '[dmap(d@3), dmap(d@5), dmap(d@7), dmap(d@9), dmap(d@11)]'
    - p_center: float = 0.01
    - per10_p_count: int = 3
    - p_bias: int = 0
    - slurm_maximum_jobs: None = 30
    - slurm_cores_per_node: int = 10
    - slurm_mem_per_job: int = 2
    - slurm_walltime: str = "1-00:00:00"
    - local_maximum_jobs: None = 10
    - force_finished: bool = False

Rotated Surface Code

d_vec = [3, 5, 7, 9, 11]

code_parameters = dict(
    code_eval = 'f"rsc(d@{c.d},p@{p})"',
    noise_eval = '"none"',
    p_center = 0.01,
    per10_p_count = 3,
    p_bias = -2,  # 0.0022
    slurm_mem_per_job = 4,
    config_vec_eval="[" + ",".join(f"dmap(d@{d})" for d in d_vec) + "]",
    slurm_maximum_jobs=30,  # smaller instances for kick starts
)

print("config_vec_eval:", code_parameters["config_vec_eval"])

folder = slugify(code_parameters["code_eval"]) + "_" + slugify(code_parameters["noise_eval"])
os.makedirs(folder, exist_ok=True)

decoder_eval_vec = [
    '"mwpm"',
    '"mwpf"',
    '"huf"',
    'f"mwpf(c@{50*c.d})"',  # auto scaling
    '"bposd"',
    '"bposd(max_iter@5)"',
    '"bposd(max_iter@5,osd_order@10,osd_method@osd_e)"',
    '"bpuf"',
    '"bpuf(max_iter@5)"',
]

jobs: list[tuple[str, str]] = []
for decoder_eval in decoder_eval_vec:
    parameters = dict(
        **code_parameters,
        decoder_eval=decoder_eval,
    )
    notebook_filename = slugify(decoder_eval) + ".ipynb"
    notebook_path = os.path.join(folder, notebook_filename)
    if os.path.exists(notebook_path):
        print("skip existing:", notebook_path)
    else:
        print("parameters:", parameters)
        pm.execute_notebook(
            template_path,
            notebook_path,
            parameters=parameters,
            prepare_only=True,  # do not run the notebook
        )
        remove_notebook_output(notebook_path)
    # create tmux session to run these commands
    tmux_name = slugify(decoder_eval) + "_" + slugify(code_parameters["code_eval"]) + "_" + slugify(code_parameters["noise_eval"])
    command = f"srun --time=23:55:00 --mem=10G --cpus-per-task=2 papermill {notebook_filename} ${{RID}}.{notebook_filename}"
    jobs.append((tmux_name, command))

script_path = os.path.join(folder, "run_all.sh")
script = tmux_list_script(jobs, prefix="RID=zz-3")
if os.path.exists(script_path):
    print("skip existing:", script_path)
else:
    with open(script_path, "w") as f:
        f.write(script)

config_vec_eval: [dmap(d@3),dmap(d@5),dmap(d@7),dmap(d@9),dmap(d@11)]
skip existing: f-rsc-d-c-d-p-p_none/mwpm.ipynb
skip existing: f-rsc-d-c-d-p-p_none/mwpf.ipynb
skip existing: f-rsc-d-c-d-p-p_none/huf.ipynb
skip existing: f-rsc-d-c-d-p-p_none/f-mwpf-c-50-c-d.ipynb
skip existing: f-rsc-d-c-d-p-p_none/bposd.ipynb
skip existing: f-rsc-d-c-d-p-p_none/bposd-max-iter-5.ipynb
skip existing: f-rsc-d-c-d-p-p_none/bposd-max-iter-5-osd-order-10-osd-method-osd-e.ipynb
skip existing: f-rsc-d-c-d-p-p_none/bpuf.ipynb
skip existing: f-rsc-d-c-d-p-p_none/bpuf-max-iter-5.ipynb
skip existing: f-rsc-d-c-d-p-p_none/run_all.sh

Un-rotated Surface Code

d_vec = [3, 5, 7, 9, 11]

code_parameters = dict(
    code_eval = 'f"usc(d@{c.d},p@{p})"',
    noise_eval = '"none"',
    p_center = 0.01,
    per10_p_count = 3,
    p_bias = -1,  # 0.0046
    slurm_mem_per_job = 4,
    config_vec_eval="[" + ",".join(f"dmap(d@{d})" for d in d_vec) + "]",
    slurm_maximum_jobs=30,  # smaller instances for kick starts
)

print("config_vec_eval:", code_parameters["config_vec_eval"])

folder = slugify(code_parameters["code_eval"]) + "_" + slugify(code_parameters["noise_eval"])
os.makedirs(folder, exist_ok=True)

decoder_eval_vec = [
    '"mwpm"',
    '"mwpf"',
    '"huf"',
    'f"mwpf(c@{50*c.d})"',  # auto scaling
    '"bposd"',
    '"bposd(max_iter@5)"',
    '"bposd(max_iter@5,osd_order@10,osd_method@osd_e)"',
    '"bpuf"',
    '"bpuf(max_iter@5)"',
]

jobs: list[tuple[str, str]] = []
for decoder_eval in decoder_eval_vec:
    parameters = dict(
        **code_parameters,
        decoder_eval=decoder_eval,
    )
    notebook_filename = slugify(decoder_eval) + ".ipynb"
    notebook_path = os.path.join(folder, notebook_filename)
    if os.path.exists(notebook_path):
        print("skip existing:", notebook_path)
    else:
        print("parameters:", parameters)
        pm.execute_notebook(
            template_path,
            notebook_path,
            parameters=parameters,
            prepare_only=True,  # do not run the notebook
        )
        remove_notebook_output(notebook_path)
    # create tmux session to run these commands
    tmux_name = slugify(decoder_eval) + "_" + slugify(code_parameters["code_eval"]) + "_" + slugify(code_parameters["noise_eval"])
    command = f"srun --time=23:55:00 --mem=10G --cpus-per-task=2 papermill {notebook_filename} ${{RID}}.{notebook_filename}"
    jobs.append((tmux_name, command))

script_path = os.path.join(folder, "run_all.sh")
script = tmux_list_script(jobs, prefix="RID=zz-3")
if os.path.exists(script_path):
    print("skip existing:", script_path)
else:
    with open(script_path, "w") as f:
        f.write(script)

config_vec_eval: [dmap(d@3),dmap(d@5),dmap(d@7),dmap(d@9),dmap(d@11)]
skip existing: f-usc-d-c-d-p-p_none/mwpm.ipynb
skip existing: f-usc-d-c-d-p-p_none/mwpf.ipynb
skip existing: f-usc-d-c-d-p-p_none/huf.ipynb
skip existing: f-usc-d-c-d-p-p_none/f-mwpf-c-50-c-d.ipynb
skip existing: f-usc-d-c-d-p-p_none/bposd.ipynb
skip existing: f-usc-d-c-d-p-p_none/bposd-max-iter-5.ipynb
skip existing: f-usc-d-c-d-p-p_none/bposd-max-iter-5-osd-order-10-osd-method-osd-e.ipynb
skip existing: f-usc-d-c-d-p-p_none/bpuf.ipynb
skip existing: f-usc-d-c-d-p-p_none/bpuf-max-iter-5.ipynb
skip existing: f-usc-d-c-d-p-p_none/run_all.sh

Repetition Code

d_vec = [3, 5, 7, 9, 11, 13, 15]

code_parameters = dict(
    code_eval = 'f"rep(d@{c.d},p@{p})"',
    noise_eval = '"none"',
    p_center = 0.1,
    per10_p_count = 3,
    p_bias = -2,  # 0.022
    slurm_mem_per_job = 4,
    config_vec_eval="[" + ",".join(f"dmap(d@{d})" for d in d_vec) + "]",
    slurm_maximum_jobs=30,  # smaller instances for kick starts
)

print("config_vec_eval:", code_parameters["config_vec_eval"])

folder = slugify(code_parameters["code_eval"]) + "_" + slugify(code_parameters["noise_eval"])
os.makedirs(folder, exist_ok=True)

decoder_eval_vec = [
    '"mwpm"',
    '"mwpf"',
    '"huf"',
    'f"mwpf(c@{50*c.d})"',  # auto scaling
    '"bposd"',
    '"bposd(max_iter@5)"',
    '"bposd(max_iter@5,osd_order@10,osd_method@osd_e)"',
    '"bpuf"',
    '"bpuf(max_iter@5)"',
]

jobs: list[tuple[str, str]] = []
for decoder_eval in decoder_eval_vec:
    parameters = dict(
        **code_parameters,
        decoder_eval=decoder_eval,
    )
    notebook_filename = slugify(decoder_eval) + ".ipynb"
    notebook_path = os.path.join(folder, notebook_filename)
    if os.path.exists(notebook_path):
        print("skip existing:", notebook_path)
    else:
        print("parameters:", parameters)
        pm.execute_notebook(
            template_path,
            notebook_path,
            parameters=parameters,
            prepare_only=True,  # do not run the notebook
        )
        remove_notebook_output(notebook_path)
    # create tmux session to run these commands
    tmux_name = slugify(decoder_eval) + "_" + slugify(code_parameters["code_eval"]) + "_" + slugify(code_parameters["noise_eval"])
    command = f"srun --time=23:55:00 --mem=10G --cpus-per-task=2 papermill {notebook_filename} ${{RID}}.{notebook_filename}"
    jobs.append((tmux_name, command))

script_path = os.path.join(folder, "run_all.sh")
script = tmux_list_script(jobs, prefix="RID=zz-3")
if os.path.exists(script_path):
    print("skip existing:", script_path)
else:
    with open(script_path, "w") as f:
        f.write(script)

config_vec_eval: [dmap(d@3),dmap(d@5),dmap(d@7),dmap(d@9),dmap(d@11),dmap(d@13),dmap(d@15)]
skip existing: f-rep-d-c-d-p-p_none/mwpm.ipynb
skip existing: f-rep-d-c-d-p-p_none/mwpf.ipynb
skip existing: f-rep-d-c-d-p-p_none/huf.ipynb
skip existing: f-rep-d-c-d-p-p_none/f-mwpf-c-50-c-d.ipynb
skip existing: f-rep-d-c-d-p-p_none/bposd.ipynb
skip existing: f-rep-d-c-d-p-p_none/bposd-max-iter-5.ipynb
skip existing: f-rep-d-c-d-p-p_none/bposd-max-iter-5-osd-order-10-osd-method-osd-e.ipynb
skip existing: f-rep-d-c-d-p-p_none/bpuf.ipynb
skip existing: f-rep-d-c-d-p-p_none/bpuf-max-iter-5.ipynb
skip existing: f-rep-d-c-d-p-p_none/run_all.sh

Color Code

d_vec = [3, 5, 7, 9, 11]

code_parameters = dict(
    code_eval = 'f"color(d@{c.d},p@{p})"',
    noise_eval = '"none"',
    p_center = 0.01,
    per10_p_count = 3,
    p_bias = -3,  # 0.001
    slurm_mem_per_job = 4,
    config_vec_eval="[" + ",".join(f"dmap(d@{d})" for d in d_vec) + "]",
    slurm_maximum_jobs=30,  # smaller instances for kick starts
)

print("config_vec_eval:", code_parameters["config_vec_eval"])

folder = slugify(code_parameters["code_eval"]) + "_" + slugify(code_parameters["noise_eval"])
os.makedirs(folder, exist_ok=True)

decoder_eval_vec = [
    '"mwpf"',
    '"huf"',
    'f"mwpf(c@{50*c.d})"',  # auto scaling
    '"bposd"',
    '"bposd(max_iter@5)"',
    '"bposd(max_iter@5,osd_order@10,osd_method@osd_e)"',
    '"bpuf"',
    '"bpuf(max_iter@5)"',
]

jobs: list[tuple[str, str]] = []
for decoder_eval in decoder_eval_vec:
    parameters = dict(
        **code_parameters,
        decoder_eval=decoder_eval,
    )
    notebook_filename = slugify(decoder_eval) + ".ipynb"
    notebook_path = os.path.join(folder, notebook_filename)
    if os.path.exists(notebook_path):
        print("skip existing:", notebook_path)
    else:
        print("parameters:", parameters)
        pm.execute_notebook(
            template_path,
            notebook_path,
            parameters=parameters,
            prepare_only=True,  # do not run the notebook
        )
        remove_notebook_output(notebook_path)
    # create tmux session to run these commands
    tmux_name = slugify(decoder_eval) + "_" + slugify(code_parameters["code_eval"]) + "_" + slugify(code_parameters["noise_eval"])
    command = f"srun --time=23:55:00 --mem=10G --cpus-per-task=2 papermill {notebook_filename} ${{RID}}.{notebook_filename}"
    jobs.append((tmux_name, command))

script_path = os.path.join(folder, "run_all.sh")
script = tmux_list_script(jobs, prefix="RID=zz-3")
if os.path.exists(script_path):
    print("skip existing:", script_path)
else:
    with open(script_path, "w") as f:
        f.write(script)

config_vec_eval: [dmap(d@3),dmap(d@5),dmap(d@7),dmap(d@9),dmap(d@11)]
skip existing: f-color-d-c-d-p-p_none/mwpf.ipynb
skip existing: f-color-d-c-d-p-p_none/huf.ipynb
skip existing: f-color-d-c-d-p-p_none/f-mwpf-c-50-c-d.ipynb
skip existing: f-color-d-c-d-p-p_none/bposd.ipynb
skip existing: f-color-d-c-d-p-p_none/bposd-max-iter-5.ipynb
skip existing: f-color-d-c-d-p-p_none/bposd-max-iter-5-osd-order-10-osd-method-osd-e.ipynb
skip existing: f-color-d-c-d-p-p_none/bpuf.ipynb
skip existing: f-color-d-c-d-p-p_none/bpuf-max-iter-5.ipynb
skip existing: f-color-d-c-d-p-p_none/run_all.sh

BB Code (Code Capacity Noise)

nkd_vec = [
    (72, 12, 6),
    (90, 8, 10),
    (108, 8, 10),
    (144, 12, 12),
    (288, 12, 18)
]

code_parameters = dict(
    code_eval = 'f"bb(n@{c.n},k@{c.k},d@{c.d})"',
    noise_eval = 'f"depolarize(p@{p})"',
    p_center = 0.01,
    per10_p_count = 6,  # need more data for this code
    p_bias = 4,  # 0.046
    slurm_mem_per_job = 4,
    config_vec_eval="[" + ",".join(f"dmap(n@{n},k@{k},d@{d})" for n, k, d in nkd_vec) + "]",
    slurm_maximum_jobs=30,  # smaller instances for kick starts
)

print("config_vec_eval:", code_parameters["config_vec_eval"])

folder = slugify(code_parameters["code_eval"]) + "_" + slugify(code_parameters["noise_eval"])
os.makedirs(folder, exist_ok=True)

decoder_eval_vec = [
    '"mwpf"',
    '"mwpf(c@1000)"',
    '"huf"',
    'f"mwpf(c@{50*c.d})"',  # auto scaling
    '"bposd"',
    '"bposd(max_iter@5)"',
    '"bposd(max_iter@5,osd_order@10,osd_method@osd_e)"',
    '"bpuf"',
    '"bpuf(max_iter@5)"',
]

jobs: list[tuple[str, str]] = []
for decoder_eval in decoder_eval_vec:
    parameters = dict(
        **code_parameters,
        decoder_eval=decoder_eval,
    )
    notebook_filename = slugify(decoder_eval) + ".ipynb"
    notebook_path = os.path.join(folder, notebook_filename)
    if os.path.exists(notebook_path):
        print("skip existing:", notebook_path)
    else:
        print("parameters:", parameters)
        pm.execute_notebook(
            template_path,
            notebook_path,
            parameters=parameters,
            prepare_only=True,  # do not run the notebook
        )
        remove_notebook_output(notebook_path)
    # create tmux session to run these commands
    tmux_name = slugify(decoder_eval) + "_" + slugify(code_parameters["code_eval"]) + "_" + slugify(code_parameters["noise_eval"])
    command = f"srun --time=23:55:00 --mem=10G --cpus-per-task=2 papermill {notebook_filename} ${{RID}}.{notebook_filename}"
    jobs.append((tmux_name, command))

script_path = os.path.join(folder, "run_all.sh")
script = tmux_list_script(jobs, prefix="RID=zz-3")
if os.path.exists(script_path):
    print("skip existing:", script_path)
else:
    with open(script_path, "w") as f:
        f.write(script)

config_vec_eval: [dmap(n@72,k@12,d@6),dmap(n@90,k@8,d@10),dmap(n@108,k@8,d@10),dmap(n@144,k@12,d@12),dmap(n@288,k@12,d@18)]
parameters: {'code_eval': 'f"bb(n@{c.n},k@{c.k},d@{c.d})"', 'noise_eval': 'f"depolarize(p@{p})"', 'p_center': 0.01, 'per10_p_count': 6, 'p_bias': 4, 'slurm_mem_per_job': 4, 'config_vec_eval': '[dmap(n@72,k@12,d@6),dmap(n@90,k@8,d@10),dmap(n@108,k@8,d@10),dmap(n@144,k@12,d@12),dmap(n@288,k@12,d@18)]', 'slurm_maximum_jobs': 30, 'decoder_eval': '"mwpf"'}
parameters: {'code_eval': 'f"bb(n@{c.n},k@{c.k},d@{c.d})"', 'noise_eval': 'f"depolarize(p@{p})"', 'p_center': 0.01, 'per10_p_count': 6, 'p_bias': 4, 'slurm_mem_per_job': 4, 'config_vec_eval': '[dmap(n@72,k@12,d@6),dmap(n@90,k@8,d@10),dmap(n@108,k@8,d@10),dmap(n@144,k@12,d@12),dmap(n@288,k@12,d@18)]', 'slurm_maximum_jobs': 30, 'decoder_eval': '"mwpf(c@1000)"'}
parameters: {'code_eval': 'f"bb(n@{c.n},k@{c.k},d@{c.d})"', 'noise_eval': 'f"depolarize(p@{p})"', 'p_center': 0.01, 'per10_p_count': 6, 'p_bias': 4, 'slurm_mem_per_job': 4, 'config_vec_eval': '[dmap(n@72,k@12,d@6),dmap(n@90,k@8,d@10),dmap(n@108,k@8,d@10),dmap(n@144,k@12,d@12),dmap(n@288,k@12,d@18)]', 'slurm_maximum_jobs': 30, 'decoder_eval': '"huf"'}
parameters: {'code_eval': 'f"bb(n@{c.n},k@{c.k},d@{c.d})"', 'noise_eval': 'f"depolarize(p@{p})"', 'p_center': 0.01, 'per10_p_count': 6, 'p_bias': 4, 'slurm_mem_per_job': 4, 'config_vec_eval': '[dmap(n@72,k@12,d@6),dmap(n@90,k@8,d@10),dmap(n@108,k@8,d@10),dmap(n@144,k@12,d@12),dmap(n@288,k@12,d@18)]', 'slurm_maximum_jobs': 30, 'decoder_eval': 'f"mwpf(c@{50*c.d})"'}
parameters: {'code_eval': 'f"bb(n@{c.n},k@{c.k},d@{c.d})"', 'noise_eval': 'f"depolarize(p@{p})"', 'p_center': 0.01, 'per10_p_count': 6, 'p_bias': 4, 'slurm_mem_per_job': 4, 'config_vec_eval': '[dmap(n@72,k@12,d@6),dmap(n@90,k@8,d@10),dmap(n@108,k@8,d@10),dmap(n@144,k@12,d@12),dmap(n@288,k@12,d@18)]', 'slurm_maximum_jobs': 30, 'decoder_eval': '"bposd"'}
parameters: {'code_eval': 'f"bb(n@{c.n},k@{c.k},d@{c.d})"', 'noise_eval': 'f"depolarize(p@{p})"', 'p_center': 0.01, 'per10_p_count': 6, 'p_bias': 4, 'slurm_mem_per_job': 4, 'config_vec_eval': '[dmap(n@72,k@12,d@6),dmap(n@90,k@8,d@10),dmap(n@108,k@8,d@10),dmap(n@144,k@12,d@12),dmap(n@288,k@12,d@18)]', 'slurm_maximum_jobs': 30, 'decoder_eval': '"bposd(max_iter@5)"'}
parameters: {'code_eval': 'f"bb(n@{c.n},k@{c.k},d@{c.d})"', 'noise_eval': 'f"depolarize(p@{p})"', 'p_center': 0.01, 'per10_p_count': 6, 'p_bias': 4, 'slurm_mem_per_job': 4, 'config_vec_eval': '[dmap(n@72,k@12,d@6),dmap(n@90,k@8,d@10),dmap(n@108,k@8,d@10),dmap(n@144,k@12,d@12),dmap(n@288,k@12,d@18)]', 'slurm_maximum_jobs': 30, 'decoder_eval': '"bposd(max_iter@5,osd_order@10,osd_method@osd_e)"'}
parameters: {'code_eval': 'f"bb(n@{c.n},k@{c.k},d@{c.d})"', 'noise_eval': 'f"depolarize(p@{p})"', 'p_center': 0.01, 'per10_p_count': 6, 'p_bias': 4, 'slurm_mem_per_job': 4, 'config_vec_eval': '[dmap(n@72,k@12,d@6),dmap(n@90,k@8,d@10),dmap(n@108,k@8,d@10),dmap(n@144,k@12,d@12),dmap(n@288,k@12,d@18)]', 'slurm_maximum_jobs': 30, 'decoder_eval': '"bpuf"'}
parameters: {'code_eval': 'f"bb(n@{c.n},k@{c.k},d@{c.d})"', 'noise_eval': 'f"depolarize(p@{p})"', 'p_center': 0.01, 'per10_p_count': 6, 'p_bias': 4, 'slurm_mem_per_job': 4, 'config_vec_eval': '[dmap(n@72,k@12,d@6),dmap(n@90,k@8,d@10),dmap(n@108,k@8,d@10),dmap(n@144,k@12,d@12),dmap(n@288,k@12,d@18)]', 'slurm_maximum_jobs': 30, 'decoder_eval': '"bpuf(max_iter@5)"'}