simply running in succession

README.md

@@ -96,6 +96,12 @@ gamma: float, optional unless beta is directly provided
 length: float, optional
     length of the fiber in m. default : 1
 
+fiber_id : int
+    in case multiple fibers are chained together, indicates the index of this particular fiber, default : 0
+
+input_transmission : float
+    number between 0 and 1 indicating how much light enters the fiber, default : 1
+
 
 ## Gas parameters
 this section is completely optional and ignored if the fiber model is "pcf"

scgenerator full anomalous/initial_config.toml

@@ -0,0 +1,32 @@
+name = "full anomalous"
+
+[fiber]
+beta = [ -1.183e-26, 8.1038e-41, -9.5205e-56, 2.0737e-70, -5.3943e-85, 1.3486e-99, -2.5495e-114, 3.0524e-129, -1.714e-144,]
+gamma = 0.11
+length = 0.02
+model = "custom"
+input_transmission = 1.0
+
+[pulse]
+power = 10000
+t0 = 2.84e-14
+shape = "gaussian"
+quantum_noise = false
+intensity_noise = 0
+
+[simulation]
+dt = 1e-15
+parallel = true
+raman_type = "measured"
+repeat = 3
+t_num = 16384
+tolerated_error = 1e-9
+z_num = 64
+behaviors = [ "spm", "ss",]
+frep = 80000000.0
+lower_wavelength_interp_limit = 3e-7
+upper_wavelength_interp_limit = 1.9e-6
+ideal_gas = false
+
+[pulse.variable]
+wavelength = [ 8.35e-7, 8.3375e-7,]

scgenerator full anomalous_1/initial_config.toml

@@ -0,0 +1,32 @@
+name = "full anomalous"
+
+[fiber]
+beta = [ -1.183e-26, 8.1038e-41, -9.5205e-56, 2.0737e-70, -5.3943e-85, 1.3486e-99, -2.5495e-114, 3.0524e-129, -1.714e-144,]
+gamma = 0.11
+length = 0.02
+model = "custom"
+input_transmission = 1.0
+
+[pulse]
+power = 10000
+t0 = 2.84e-14
+shape = "gaussian"
+quantum_noise = false
+intensity_noise = 0
+
+[simulation]
+dt = 1e-15
+parallel = true
+raman_type = "measured"
+repeat = 3
+t_num = 16384
+tolerated_error = 1e-9
+z_num = 64
+behaviors = [ "spm", "ss",]
+frep = 80000000.0
+lower_wavelength_interp_limit = 3e-7
+upper_wavelength_interp_limit = 1.9e-6
+ideal_gas = false
+
+[pulse.variable]
+wavelength = [ 8.35e-7, 8.3375e-7,]

src/scgenerator/cli/cli.py

@@ -4,7 +4,10 @@ import random
 import sys
 
 import ray
-from scgenerator.physics.simulate import new_simulations, resume_simulations, SequencialSimulations
+
+from scgenerator import initialize
+from ..physics.simulate import new_simulations, resume_simulations, SequencialSimulations
+from .. import io
 
 
 def create_parser():
@@ -32,7 +35,13 @@ def create_parser():
 
     run_parser = subparsers.add_parser("run", help="run a simulation from a config file")
 
-    run_parser.add_argument("config", help="path to the toml configuration file")
+    run_parser.add_argument("configs", help="path(s) to the toml configuration file(s)", nargs="+")
+    run_parser.add_argument(
+        "appendto",
+        help="optional directory where a compatible simulation has already been ran",
+        nargs="?",
+        default=None,
+    )
     run_parser.set_defaults(func=run_sim)
 
     resume_parser = subparsers.add_parser("resume", help="resume a simulation")
@@ -42,6 +51,16 @@ def create_parser():
     )
     resume_parser.set_defaults(func=resume_sim)
 
+    merge_parser = subparsers.add_parser("merge", help="merge simulation results")
+    merge_parser.add_argument(
+        "paths",
+        nargs="+",
+        help="path(s) to simulation folder(s) containing 'initial_config.toml'. If more "
+        "than one path is given, simulations are appended to each other as if they're "
+        "physically the continuation of the previous one.",
+    )
+    merge_parser.set_defaults(func=merge)
+
     return parser
 
 
@@ -54,26 +73,49 @@ def main():
 def run_sim(args):
 
     method = prep_ray(args)
-    sim = new_simulations(args.config, args.id, Method=method)
+    configs = args.configs.copy()
+    first_config = configs.pop(0)
+    if args.appendto is None:
+        sim = new_simulations(first_config, args.id, method=method)
+    else:
+        sim = new_simulations(
+            first_config, args.id, data_folder=args.appendto, method=method, initial=False
+        )
     sim.run()
+    data_folders = [sim.data_folder]
+    for config in configs:
+        print("launching", config)
+        sim = new_simulations(
+            config, args.id, data_folder=data_folders[-1], method=method, initial=False
+        )
+        sim.run()
+        data_folders.append(sim.data_folder)
+    io.merge(data_folders)
+
+
+def merge(args):
+    io.merge(args.paths)
 
 
 def prep_ray(args):
     if args.start_ray:
         init_str = ray.init()
     elif not args.no_ray:
-        init_str = ray.init(
+        try:
-            address="auto",
+            init_str = ray.init(
-            # _node_ip_address=os.environ.get("ip_head", "127.0.0.1").split(":")[0],
+                address="auto",
-            _redis_password=os.environ.get("redis_password", "caco1234"),
+                # _node_ip_address=os.environ.get("ip_head", "127.0.0.1").split(":")[0],
-        )
+                _redis_password=os.environ.get("redis_password", "caco1234"),
-        print(init_str)
+            )
+            print(init_str)
+        except ConnectionError:
+            pass
     return SequencialSimulations if args.no_ray else None
 
 
 def resume_sim(args):
     method = prep_ray(args)
-    sim = resume_simulations(args.data_dir, args.id, Method=method)
+    sim = resume_simulations(args.data_dir, args.id, method=method)
     sim.run()
 
 

src/scgenerator/const.py

@@ -108,6 +108,7 @@ valid_param_types = dict(
         name=lambda s: isinstance(s, str),
     ),
     fiber=dict(
+        input_transmission=in_range_incl(num, (0, 1)),
         gamma=num,
         pitch=in_range_excl(num, (0, 1e-3)),
         pitch_ratio=in_range_excl(num, (0, 1)),
@@ -190,6 +191,7 @@ valid_variable = dict(
         "capillary_nested",
         "he_mode",
         "fit_parameters",
+        "input_transmission",
     ],
     gas=["pressure", "temperature", "gas_name", "plasma_density"],
     pulse=[

src/scgenerator/defaults.py

@@ -3,6 +3,7 @@ import matplotlib.pyplot as plt
 from .errors import MissingParameterError
 
 default_parameters = dict(
+    input_transmission=1.0,
     name="no name",
     he_mode=(1, 1),
     fit_parameters=(0.08, 200e-9),

src/scgenerator/initialize.py

@@ -1,6 +1,6 @@
 import os
 from collections.abc import Mapping
-from typing import Any, Iterator, List, Tuple
+from typing import Any, Dict, Iterator, List, Tuple
 
 import numpy as np
 from numpy import pi
@@ -25,7 +25,7 @@ class ParamSequence(Mapping):
         self.num_steps = self.num_sim * self.config["simulation"]["z_num"]
         self.single_sim = self.num_sim == 1
 
-    def __iter__(self) -> Iterator[Tuple[List[Tuple[str, Any]], dict]]:
+    def __iter__(self) -> Iterator[Tuple[List[Tuple[str, Any]], Dict[str, Any]]]:
         """iterates through all possible parameters, yielding a config as well as a flattened
         computed parameters set each time"""
         for variable_list, full_config in required_simulations(self.config):
@@ -41,6 +41,22 @@ class ParamSequence(Mapping):
         return f"dispatcher generated from config {self.name}"
 
 
+class ContinuationParamSequence(ParamSequence):
+    def __init__(self, folder: str, new_config: Dict[str, Any]):
+        self.path = folder
+        init_config = io.load_previous_parameters(os.path.join(self.path, "initial_config.toml"))
+        new_config = utils.deep_update(init_config, new_config)
+        super().__init__(new_config)
+
+    def __iter__(self) -> Iterator[Tuple[List[Tuple[str, Any]], Dict[str, Any]]]:
+        """iterates through all possible parameters, yielding a config as well as a flattened
+        computed parameters set each time"""
+        for variable_list, full_config in required_simulations(self.config):
+            sim_folder = os.path.join(self.path, utils.format_variable_list(variable_list))
+
+            yield variable_list, compute_subsequent_paramters(sim_folder, full_config, self.config)
+
+
 class RecoveryParamSequence(ParamSequence):
     def __init__(self, config, task_id):
         super().__init__(config)
@@ -166,7 +182,7 @@ def tspace(time_window=None, t_num=None, dt=None):
         raise TypeError("not enough parameter to determine time vector")
 
 
-def validate_single_parameter(section, key, value):
+def validate_single_parameter(section: str, key: str, value: Any):
     try:
         func = valid_param_types[section][key]
     except KeyError:
@@ -273,8 +289,8 @@ def _ensure_consistency_fiber(fiber):
         else:
             for param in hc_model_specific_parameters[fiber["model"]]:
                 fiber = defaults.get_fiber(fiber, param)
-
+    for param in ["length", "input_transmission"]:
-    fiber = defaults.get(fiber, "length")
+        fiber = defaults.get(fiber, param)
     return fiber
 
 
@@ -402,6 +418,7 @@ def _ensure_consistency(config):
 
     # ensure every required parameter has a value
     config["name"] = config.get("name", "no name")
+
     config["fiber"] = _ensure_consistency_fiber(config.get("fiber", {}))
 
     if config["fiber"]["model"] in hc_model_specific_parameters:
@@ -425,7 +442,7 @@ def recover_params(params: dict, variable_only: List[Tuple[str, Any]], task_id:
     return params
 
 
-def compute_init_parameters(config):
+def compute_init_parameters(config: Dict[str, Any]) -> Dict[str, Any]:
     """computes all derived values from a config dictionary
 
     Parameters
@@ -515,6 +532,22 @@ def compute_init_parameters(config):
     return params
 
 
+def compute_subsequent_paramters(
+    sim_folder: str, init_config: Dict[str, Any], new_config: Dict[str, Any]
+) -> Dict[str, Any]:
+
+    init_config["fiber"] = new_config["fiber"]
+    init_config["simulation"]["z_num"] = new_config.get("simulation", init_config["simulation"])[
+        "z_num"
+    ]
+
+    params = compute_init_parameters(init_config)
+    params["spec_0"] = io.load_last_spectrum(sim_folder)[1]
+    params["field_0"] = np.fft.ifft(params["spec_0"]) * params["input_transmission"]
+
+    return params
+
+
 def _update_pulse_parameters(params):
     (
         params["width"],

src/scgenerator/io.py

@@ -5,11 +5,13 @@ from glob import glob
 from typing import Any, Dict, Iterable, List, Tuple
 
 import numpy as np
+from numpy.lib import delete
 import pkg_resources as pkg
 import toml
 from ray import util
 from send2trash import TrashPermissionError, send2trash
 from tqdm import tqdm
+from pathlib import Path
 
 from . import utils
 from .const import ENVIRON_KEY_BASE, PARAM_SEPARATOR, PREFIX_KEY_BASE, TMP_FOLDER_KEY_BASE
@@ -358,12 +360,53 @@ def find_last_spectrum_file(path: str):
             return num - 1
 
 
-def load_last_spectrum(path: str):
+def load_last_spectrum(path: str) -> Tuple[int, np.ndarray]:
     num = find_last_spectrum_file(path)
     return num, np.load(os.path.join(path, f"spectrum_{num}.npy"))
 
 
-def merge_same_simulations(path: str):
+def merge(paths: List[str]):
+    for path in paths:
+        merge_same_simulations(path, delete=False)
+
+    if len(paths) < 2:
+        return
+
+    append_simulations(paths)
+
+
+def append_simulations(paths: List[os.PathLike]):
+    paths: List[Path] = [Path(p).resolve() for p in paths]
+    master_sim = paths[0]
+    merged_path = master_sim.parent / "merged_sims"
+    merged_path.mkdir(exist_ok=True)
+    for i, path in enumerate(paths):
+        shutil.copy(path / "initial_config.toml", merged_path / f"initial_config{i}.toml")
+    for sim in master_sim.glob("*"):
+        if not sim.is_dir() or not str(sim).endswith("merged"):
+            continue
+        sim_name = sim.name
+        merge_sim_path = merged_path / sim_name
+        merge_sim_path.mkdir(exist_ok=True)
+        shutil.copy(sim / "params.toml", merge_sim_path / f"params.toml")
+        z = []
+        z_num = 0
+        last_z = 0
+        for path in paths:
+            curr_z_num = load_toml(str(path / sim_name / "params.toml"))["z_num"]
+            for i in range(curr_z_num):
+                shutil.copy(
+                    path / sim_name / f"spectra_{i}.npy",
+                    merge_sim_path / f"spectra_{i + z_num}.npy",
+                )
+            z_arr = np.load(path / sim_name / "z.npy")
+            z.append(z_arr + last_z)
+            last_z += z_arr[-1]
+            z_num += curr_z_num
+        np.save(merge_sim_path / "z.npy", np.concatenate(z))
+
+
+def merge_same_simulations(path: str, delete=True):
     logger = get_logger(__name__)
     num_separator = PARAM_SEPARATOR + "num" + PARAM_SEPARATOR
     sub_folders = get_data_subfolders(path)
@@ -399,7 +442,10 @@ def merge_same_simulations(path: str):
 
             # write new files only once all those from one parameter set are collected
             if repeat_id == max_repeat_id:
-                out_path = os.path.join(path, utils.format_variable_list(variable_and_ind[:-1]))
+                out_path = os.path.join(
+                    path,
+                    utils.format_variable_list(variable_and_ind[:-1]) + PARAM_SEPARATOR + "merged",
+                )
                 out_path = ensure_folder(out_path, prevent_overwrite=False)
                 spectra = np.array(spectra).reshape(repeat, len(spectra[0]))
                 np.save(os.path.join(out_path, f"spectra_{z_id}.npy"), spectra.squeeze())
@@ -413,11 +459,12 @@ def merge_same_simulations(path: str):
                         )
     pbar.close()
 
-    try:
+    if delete:
-        for sub_folder in sub_folders:
+        try:
-            send2trash(sub_folder)
+            for sub_folder in sub_folders:
-    except TrashPermissionError:
+                send2trash(sub_folder)
-        logger.warning(f"could not send send {len(base_folders)} folder(s) to trash")
+        except TrashPermissionError:
+            logger.warning(f"could not send send {len(base_folders)} folder(s) to trash")
 
 
 def get_data_folder(task_id: int, name_if_new: str = "data"):

src/scgenerator/physics/simulate.py

@@ -130,8 +130,8 @@ class RK4IP:
 
         # Initial setup of simulation parameters
         self.d_w = self.w_c[1] - self.w_c[0]  # resolution of the frequency grid
-        self.z = self.z_targets.pop(0)
         self.z_stored = list(self.z_targets.copy()[0 : self.starting_num + 1])
+        self.z = self.z_targets.pop(0)
 
         # Setup initial values for every physical quantity that we want to track
         self.current_spectrum = self.spec_0.copy()
@@ -416,11 +416,6 @@ class Simulations:
         self._run_available()
         self.ensure_finised_and_complete()
 
-        self.logger.info(f"Merging data...")
-        self.merge_data()
-
-        self.logger.info(f"Finished simulations from config {self.name} !")
-
     def _run_available(self):
         for variable, params in self.param_seq:
             io.save_parameters(
@@ -456,9 +451,6 @@ class Simulations:
     def stop(self):
         raise NotImplementedError()
 
-    def merge_data(self):
-        io.merge_same_simulations(self.data_folder)
-
 
 class SequencialSimulations(Simulations, available=True, priority=0):
     def new_sim(self, variable_list: List[tuple], params: Dict[str, Any]):
@@ -473,7 +465,7 @@ class SequencialSimulations(Simulations, available=True, priority=0):
         pass
 
 
-class MultiProcSimulations(Simulations, available=True, priority=1):
+class MultiProcSimulations(Simulations, available=True, priority=10):
     def __init__(self, param_seq: initialize.ParamSequence, task_id, data_folder):
         super().__init__(param_seq, task_id=task_id, data_folder=data_folder)
         self.sim_jobs_per_node = max(1, os.cpu_count() // 2)
@@ -671,34 +663,40 @@ def new_simulations(
     config_file: str,
     task_id: int,
     data_folder="scgenerator/",
-    Method: Type[Simulations] = None,
+    method: Type[Simulations] = None,
+    initial=True,
 ) -> Simulations:
 
     config = io.load_toml(config_file)
-    param_seq = initialize.ParamSequence(config)
+    if initial:
+        param_seq = initialize.ParamSequence(config)
+    else:
+        param_seq = initialize.ContinuationParamSequence(data_folder, config)
 
-    return _new_simulations(param_seq, task_id, data_folder, Method)
+    print(f"{param_seq.name=}")
+
+    return _new_simulations(param_seq, task_id, data_folder, method)
 
 
 def resume_simulations(
-    data_folder: str, task_id: int = 0, Method: Type[Simulations] = None
+    data_folder: str, task_id: int = 0, method: Type[Simulations] = None
 ) -> Simulations:
 
     config = io.load_toml(os.path.join(data_folder, "initial_config.toml"))
     io.set_data_folder(task_id, data_folder)
     param_seq = initialize.RecoveryParamSequence(config, task_id)
 
-    return _new_simulations(param_seq, task_id, data_folder, Method)
+    return _new_simulations(param_seq, task_id, data_folder, method)
 
 
 def _new_simulations(
     param_seq: initialize.ParamSequence,
     task_id,
     data_folder,
-    Method: Type[Simulations],
+    method: Type[Simulations],
 ) -> Simulations:
-    if Method is not None:
+    if method is not None:
-        return Method(param_seq, task_id, data_folder=data_folder)
+        return method(param_seq, task_id, data_folder=data_folder)
     elif param_seq.num_sim > 1 and param_seq["simulation", "parallel"] and using_ray:
         return Simulations.get_best_method()(param_seq, task_id, data_folder=data_folder)
     else:

src/scgenerator/plotting.py

@@ -7,7 +7,7 @@ from matplotlib.colors import ListedColormap
 from scipy.interpolate import UnivariateSpline
 
 from . import io, math
-from .math import abs2, make_uniform_1D, span
+from .math import abs2, length, make_uniform_1D, span
 from .physics import pulse, units
 from .defaults import default_plotting as defaults
 
@@ -296,7 +296,7 @@ def _finish_plot_2D(
 
     folder_name = ""
     if is_new_plot:
-        folder_name, file_name, fig, ax = io.plot_setup(
+        folder_name, file_name, fig, ax = plot_setup(
             file_name=file_name, file_type=file_type, params=params
         )
     else:
@@ -549,11 +549,22 @@ def plot_results_2D(
             [make_uniform_1D(v, x_axis, n=len(x_axis), method="linear") for v in values]
         )
 
+    z = params["z_targets"]
+    lim_diff = 1e-5 * np.max(z)
+    dz_s = np.diff(z)
+    if not np.all(np.diff(dz_s) < lim_diff):
+        new_z = np.linspace(
+            *span(z), int(np.floor((np.max(z) - np.min(z)) / np.min(dz_s[dz_s > lim_diff])))
+        )
+        values = np.array(
+            [make_uniform_1D(v, z, n=len(new_z), method="linear") for v in values.T]
+        ).T
+        z = new_z
     return _finish_plot_2D(
         values,
         x_axis,
         plt_range[2].label,
-        params["z_targets"],
+        z,
         "propagation distance (m)",
         log,
         vmin,
@@ -681,7 +692,7 @@ def plot_results_1D(
 
     folder_name = ""
     if is_new_plot:
-        folder_name, file_name, fig, ax = io.plot_setup(
+        folder_name, file_name, fig, ax = plot_setup(
             file_name=file_name, file_type=file_type, params=params
         )
     else:
@@ -845,11 +856,11 @@ def plot_avg(
     if is_new_plot:
         if add_coherence:
             mode = "coherence_T" if transpose else "coherence"
-            folder_name, file_name, fig, (top, bot) = io.plot_setup(
+            folder_name, file_name, fig, (top, bot) = plot_setup(
                 file_name=file_name, file_type=file_type, params=params, mode=mode
             )
         else:
-            folder_name, file_name, fig, top = io.plot_setup(
+            folder_name, file_name, fig, top = plot_setup(
                 file_name=file_name, file_type=file_type, params=params
             )
             bot = top

src/scgenerator/spectra.py

@@ -60,6 +60,8 @@ class Pulse(Sequence):
         self.w_order = np.argsort(w)
         self.w = w
         self.wl = units.m.inv(self.w)
+        self.params["w"] = self.w
+        self.params["z_targets"] = self.z
 
     def __iter__(self):
         """

src/scgenerator/utils.py

@@ -5,12 +5,13 @@ scgenerator module but some function may be used in any python program
 """
 
 
+import collections
 import datetime as dt
 import itertools
 import logging
 import re
 import socket
-from typing import Any, Callable, Iterator, List, Tuple, Union
+from typing import Any, Callable, Iterator, List, Mapping, Tuple, Union
 from asyncio import Event
 
 import numpy as np
@@ -337,6 +338,15 @@ def parallelize(func, arg_iter, sim_jobs=4, progress_tracker_kwargs=None, const_
     return np.array(results)
 
 
+def deep_update(d: Mapping, u: Mapping):
+    for k, v in u.items():
+        if isinstance(v, collections.abc.Mapping):
+            d[k] = deep_update(d.get(k, {}), v)
+        else:
+            d[k] = v
+    return d
+
+
 def formatted_hostname():
     s = socket.gethostname().replace(".", "_")
     return (PREFIX_KEY_BASE + s).upper()