dedebenui/scgenerator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

simply running in succession

Browse Source
This commit is contained in:
Benoît Sierro
2021-05-25 12:38:23 +02:00
parent d34c3e6c5d
commit fcacba1cc5
14 changed files with 343 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
README.md
View File

@@ -96,6 +96,12 @@ gamma: float, optional unless beta is directly provided
length: float, optional
length of the fiber in m. default : 1
fiber_id : int
in case multiple fibers are chained together, indicates the index of this particular fiber, default : 0
input_transmission : float
number between 0 and 1 indicating how much light enters the fiber, default : 1
## Gas parameters
this section is completely optional and ignored if the fiber model is "pcf"

39
scgenerator full anomalous/id 0 wavelength 8.35e-07 num 0/params.toml Normal file
View File

File diff suppressed because one or more lines are too long

32
scgenerator full anomalous/initial_config.toml Normal file
View File

@@ -0,0 +1,32 @@
name = "full anomalous"
[fiber]
beta = [ -1.183e-26, 8.1038e-41, -9.5205e-56, 2.0737e-70, -5.3943e-85, 1.3486e-99, -2.5495e-114, 3.0524e-129, -1.714e-144,]
gamma = 0.11
length = 0.02
model = "custom"
input_transmission = 1.0
[pulse]
power = 10000
t0 = 2.84e-14
shape = "gaussian"
quantum_noise = false
intensity_noise = 0
[simulation]
dt = 1e-15
parallel = true
raman_type = "measured"
repeat = 3
t_num = 16384
tolerated_error = 1e-9
z_num = 64
behaviors = [ "spm", "ss",]
frep = 80000000.0
lower_wavelength_interp_limit = 3e-7
upper_wavelength_interp_limit = 1.9e-6
ideal_gas = false
[pulse.variable]
wavelength = [ 8.35e-7, 8.3375e-7,]

39
scgenerator full anomalous_1/id 0 wavelength 8.35e-07 num 0/params.toml Normal file
View File

File diff suppressed because one or more lines are too long

32
scgenerator full anomalous_1/initial_config.toml Normal file
View File

@@ -0,0 +1,32 @@
name = "full anomalous"
[fiber]
beta = [ -1.183e-26, 8.1038e-41, -9.5205e-56, 2.0737e-70, -5.3943e-85, 1.3486e-99, -2.5495e-114, 3.0524e-129, -1.714e-144,]
gamma = 0.11
length = 0.02
model = "custom"
input_transmission = 1.0
[pulse]
power = 10000
t0 = 2.84e-14
shape = "gaussian"
quantum_noise = false
intensity_noise = 0
[simulation]
dt = 1e-15
parallel = true
raman_type = "measured"
repeat = 3
t_num = 16384
tolerated_error = 1e-9
z_num = 64
behaviors = [ "spm", "ss",]
frep = 80000000.0
lower_wavelength_interp_limit = 3e-7
upper_wavelength_interp_limit = 1.9e-6
ideal_gas = false
[pulse.variable]
wavelength = [ 8.35e-7, 8.3375e-7,]

62
src/scgenerator/cli/cli.py
View File

@@ -4,7 +4,10 @@ import random
import sys
import ray
from scgenerator.physics.simulate import new_simulations, resume_simulations, SequencialSimulations
from scgenerator import initialize
from ..physics.simulate import new_simulations, resume_simulations, SequencialSimulations
from .. import io
def create_parser():
@@ -32,7 +35,13 @@ def create_parser():
run_parser = subparsers.add_parser("run", help="run a simulation from a config file")
run_parser.add_argument("config", help="path to the toml configuration file")
run_parser.add_argument("configs", help="path(s) to the toml configuration file(s)", nargs="+")
run_parser.add_argument(
"appendto",
help="optional directory where a compatible simulation has already been ran",
nargs="?",
default=None,
)
run_parser.set_defaults(func=run_sim)
resume_parser = subparsers.add_parser("resume", help="resume a simulation")
@@ -42,6 +51,16 @@ def create_parser():
)
resume_parser.set_defaults(func=resume_sim)
merge_parser = subparsers.add_parser("merge", help="merge simulation results")
merge_parser.add_argument(
"paths",
nargs="+",
help="path(s) to simulation folder(s) containing 'initial_config.toml'. If more "
"than one path is given, simulations are appended to each other as if they're "
"physically the continuation of the previous one.",
)
merge_parser.set_defaults(func=merge)
return parser
@@ -54,26 +73,49 @@ def main():
def run_sim(args):
method = prep_ray(args)
sim = new_simulations(args.config, args.id, Method=method)
configs = args.configs.copy()
first_config = configs.pop(0)
if args.appendto is None:
sim = new_simulations(first_config, args.id, method=method)
else:
sim = new_simulations(
first_config, args.id, data_folder=args.appendto, method=method, initial=False
)
sim.run()
data_folders = [sim.data_folder]
for config in configs:
print("launching", config)
sim = new_simulations(
config, args.id, data_folder=data_folders[-1], method=method, initial=False
)
sim.run()
data_folders.append(sim.data_folder)
io.merge(data_folders)
def merge(args):
io.merge(args.paths)
def prep_ray(args):
if args.start_ray:
init_str = ray.init()
elif not args.no_ray:
init_str = ray.init(
address="auto",
# _node_ip_address=os.environ.get("ip_head", "127.0.0.1").split(":")[0],
_redis_password=os.environ.get("redis_password", "caco1234"),
)
print(init_str)
try:
init_str = ray.init(
address="auto",
# _node_ip_address=os.environ.get("ip_head", "127.0.0.1").split(":")[0],
_redis_password=os.environ.get("redis_password", "caco1234"),
)
print(init_str)
except ConnectionError:
pass
return SequencialSimulations if args.no_ray else None
def resume_sim(args):
method = prep_ray(args)
sim = resume_simulations(args.data_dir, args.id, Method=method)
sim = resume_simulations(args.data_dir, args.id, method=method)
sim.run()

2
src/scgenerator/const.py
View File

@@ -108,6 +108,7 @@ valid_param_types = dict(
name=lambda s: isinstance(s, str),
),
fiber=dict(
input_transmission=in_range_incl(num, (0, 1)),
gamma=num,
pitch=in_range_excl(num, (0, 1e-3)),
pitch_ratio=in_range_excl(num, (0, 1)),
@@ -190,6 +191,7 @@ valid_variable = dict(
"capillary_nested",
"he_mode",
"fit_parameters",
"input_transmission",
],
gas=["pressure", "temperature", "gas_name", "plasma_density"],
pulse=[

1
src/scgenerator/defaults.py
View File

@@ -3,6 +3,7 @@ import matplotlib.pyplot as plt
from .errors import MissingParameterError
default_parameters = dict(
input_transmission=1.0,
name="no name",
he_mode=(1, 1),
fit_parameters=(0.08, 200e-9),

45
src/scgenerator/initialize.py
View File

@@ -1,6 +1,6 @@
import os
from collections.abc import Mapping
from typing import Any, Iterator, List, Tuple
from typing import Any, Dict, Iterator, List, Tuple
import numpy as np
from numpy import pi
@@ -25,7 +25,7 @@ class ParamSequence(Mapping):
self.num_steps = self.num_sim * self.config["simulation"]["z_num"]
self.single_sim = self.num_sim == 1
def __iter__(self) -> Iterator[Tuple[List[Tuple[str, Any]], dict]]:
def __iter__(self) -> Iterator[Tuple[List[Tuple[str, Any]], Dict[str, Any]]]:
"""iterates through all possible parameters, yielding a config as well as a flattened
computed parameters set each time"""
for variable_list, full_config in required_simulations(self.config):
@@ -41,6 +41,22 @@ class ParamSequence(Mapping):
return f"dispatcher generated from config {self.name}"
class ContinuationParamSequence(ParamSequence):
def __init__(self, folder: str, new_config: Dict[str, Any]):
self.path = folder
init_config = io.load_previous_parameters(os.path.join(self.path, "initial_config.toml"))
new_config = utils.deep_update(init_config, new_config)
super().__init__(new_config)
def __iter__(self) -> Iterator[Tuple[List[Tuple[str, Any]], Dict[str, Any]]]:
"""iterates through all possible parameters, yielding a config as well as a flattened
computed parameters set each time"""
for variable_list, full_config in required_simulations(self.config):
sim_folder = os.path.join(self.path, utils.format_variable_list(variable_list))
yield variable_list, compute_subsequent_paramters(sim_folder, full_config, self.config)
class RecoveryParamSequence(ParamSequence):
def __init__(self, config, task_id):
super().__init__(config)
@@ -166,7 +182,7 @@ def tspace(time_window=None, t_num=None, dt=None):
raise TypeError("not enough parameter to determine time vector")
def validate_single_parameter(section, key, value):
def validate_single_parameter(section: str, key: str, value: Any):
try:
func = valid_param_types[section][key]
except KeyError:
@@ -273,8 +289,8 @@ def _ensure_consistency_fiber(fiber):
else:
for param in hc_model_specific_parameters[fiber["model"]]:
fiber = defaults.get_fiber(fiber, param)
fiber = defaults.get(fiber, "length")
for param in ["length", "input_transmission"]:
fiber = defaults.get(fiber, param)
return fiber
@@ -402,6 +418,7 @@ def _ensure_consistency(config):
# ensure every required parameter has a value
config["name"] = config.get("name", "no name")
config["fiber"] = _ensure_consistency_fiber(config.get("fiber", {}))
if config["fiber"]["model"] in hc_model_specific_parameters:
@@ -425,7 +442,7 @@ def recover_params(params: dict, variable_only: List[Tuple[str, Any]], task_id:
return params
def compute_init_parameters(config):
def compute_init_parameters(config: Dict[str, Any]) -> Dict[str, Any]:
"""computes all derived values from a config dictionary
Parameters
@@ -515,6 +532,22 @@ def compute_init_parameters(config):
return params
def compute_subsequent_paramters(
sim_folder: str, init_config: Dict[str, Any], new_config: Dict[str, Any]
) -> Dict[str, Any]:
init_config["fiber"] = new_config["fiber"]
init_config["simulation"]["z_num"] = new_config.get("simulation", init_config["simulation"])[
"z_num"
]
params = compute_init_parameters(init_config)
params["spec_0"] = io.load_last_spectrum(sim_folder)[1]
params["field_0"] = np.fft.ifft(params["spec_0"]) * params["input_transmission"]
return params
def _update_pulse_parameters(params):
(
params["width"],

63
src/scgenerator/io.py
View File

@@ -5,11 +5,13 @@ from glob import glob
from typing import Any, Dict, Iterable, List, Tuple
import numpy as np
from numpy.lib import delete
import pkg_resources as pkg
import toml
from ray import util
from send2trash import TrashPermissionError, send2trash
from tqdm import tqdm
from pathlib import Path
from . import utils
from .const import ENVIRON_KEY_BASE, PARAM_SEPARATOR, PREFIX_KEY_BASE, TMP_FOLDER_KEY_BASE
@@ -358,12 +360,53 @@ def find_last_spectrum_file(path: str):
return num - 1
def load_last_spectrum(path: str):
def load_last_spectrum(path: str) -> Tuple[int, np.ndarray]:
num = find_last_spectrum_file(path)
return num, np.load(os.path.join(path, f"spectrum_{num}.npy"))
def merge_same_simulations(path: str):
def merge(paths: List[str]):
for path in paths:
merge_same_simulations(path, delete=False)
if len(paths) < 2:
return
append_simulations(paths)
def append_simulations(paths: List[os.PathLike]):
paths: List[Path] = [Path(p).resolve() for p in paths]
master_sim = paths[0]
merged_path = master_sim.parent / "merged_sims"
merged_path.mkdir(exist_ok=True)
for i, path in enumerate(paths):
shutil.copy(path / "initial_config.toml", merged_path / f"initial_config{i}.toml")
for sim in master_sim.glob("*"):
if not sim.is_dir() or not str(sim).endswith("merged"):
continue
sim_name = sim.name
merge_sim_path = merged_path / sim_name
merge_sim_path.mkdir(exist_ok=True)
shutil.copy(sim / "params.toml", merge_sim_path / f"params.toml")
z = []
z_num = 0
last_z = 0
for path in paths:
curr_z_num = load_toml(str(path / sim_name / "params.toml"))["z_num"]
for i in range(curr_z_num):
shutil.copy(
path / sim_name / f"spectra_{i}.npy",
merge_sim_path / f"spectra_{i + z_num}.npy",
)
z_arr = np.load(path / sim_name / "z.npy")
z.append(z_arr + last_z)
last_z += z_arr[-1]
z_num += curr_z_num
np.save(merge_sim_path / "z.npy", np.concatenate(z))
def merge_same_simulations(path: str, delete=True):
logger = get_logger(__name__)
num_separator = PARAM_SEPARATOR + "num" + PARAM_SEPARATOR
sub_folders = get_data_subfolders(path)
@@ -399,7 +442,10 @@ def merge_same_simulations(path: str):
# write new files only once all those from one parameter set are collected
if repeat_id == max_repeat_id:
out_path = os.path.join(path, utils.format_variable_list(variable_and_ind[:-1]))
out_path = os.path.join(
path,
utils.format_variable_list(variable_and_ind[:-1]) + PARAM_SEPARATOR + "merged",
)
out_path = ensure_folder(out_path, prevent_overwrite=False)
spectra = np.array(spectra).reshape(repeat, len(spectra[0]))
np.save(os.path.join(out_path, f"spectra_{z_id}.npy"), spectra.squeeze())
@@ -413,11 +459,12 @@ def merge_same_simulations(path: str):
)
pbar.close()
try:
for sub_folder in sub_folders:
send2trash(sub_folder)
except TrashPermissionError:
logger.warning(f"could not send send {len(base_folders)} folder(s) to trash")
if delete:
try:
for sub_folder in sub_folders:
send2trash(sub_folder)
except TrashPermissionError:
logger.warning(f"could not send send {len(base_folders)} folder(s) to trash")
def get_data_folder(task_id: int, name_if_new: str = "data"):

34
src/scgenerator/physics/simulate.py
View File

@@ -130,8 +130,8 @@ class RK4IP:
# Initial setup of simulation parameters
self.d_w = self.w_c[1] - self.w_c[0] # resolution of the frequency grid
self.z = self.z_targets.pop(0)
self.z_stored = list(self.z_targets.copy()[0 : self.starting_num + 1])
self.z = self.z_targets.pop(0)
# Setup initial values for every physical quantity that we want to track
self.current_spectrum = self.spec_0.copy()
@@ -416,11 +416,6 @@ class Simulations:
self._run_available()
self.ensure_finised_and_complete()
self.logger.info(f"Merging data...")
self.merge_data()
self.logger.info(f"Finished simulations from config {self.name} !")
def _run_available(self):
for variable, params in self.param_seq:
io.save_parameters(
@@ -456,9 +451,6 @@ class Simulations:
def stop(self):
raise NotImplementedError()
def merge_data(self):
io.merge_same_simulations(self.data_folder)
class SequencialSimulations(Simulations, available=True, priority=0):
def new_sim(self, variable_list: List[tuple], params: Dict[str, Any]):
@@ -473,7 +465,7 @@ class SequencialSimulations(Simulations, available=True, priority=0):
pass
class MultiProcSimulations(Simulations, available=True, priority=1):
class MultiProcSimulations(Simulations, available=True, priority=10):
def __init__(self, param_seq: initialize.ParamSequence, task_id, data_folder):
super().__init__(param_seq, task_id=task_id, data_folder=data_folder)
self.sim_jobs_per_node = max(1, os.cpu_count() // 2)
@@ -671,34 +663,40 @@ def new_simulations(
config_file: str,
task_id: int,
data_folder="scgenerator/",
Method: Type[Simulations] = None,
method: Type[Simulations] = None,
initial=True,
) -> Simulations:
config = io.load_toml(config_file)
param_seq = initialize.ParamSequence(config)
if initial:
param_seq = initialize.ParamSequence(config)
else:
param_seq = initialize.ContinuationParamSequence(data_folder, config)
return _new_simulations(param_seq, task_id, data_folder, Method)
print(f"{param_seq.name=}")
return _new_simulations(param_seq, task_id, data_folder, method)
def resume_simulations(
data_folder: str, task_id: int = 0, Method: Type[Simulations] = None
data_folder: str, task_id: int = 0, method: Type[Simulations] = None
) -> Simulations:
config = io.load_toml(os.path.join(data_folder, "initial_config.toml"))
io.set_data_folder(task_id, data_folder)
param_seq = initialize.RecoveryParamSequence(config, task_id)
return _new_simulations(param_seq, task_id, data_folder, Method)
return _new_simulations(param_seq, task_id, data_folder, method)
def _new_simulations(
param_seq: initialize.ParamSequence,
task_id,
data_folder,
Method: Type[Simulations],
method: Type[Simulations],
) -> Simulations:
if Method is not None:
return Method(param_seq, task_id, data_folder=data_folder)
if method is not None:
return method(param_seq, task_id, data_folder=data_folder)
elif param_seq.num_sim > 1 and param_seq["simulation", "parallel"] and using_ray:
return Simulations.get_best_method()(param_seq, task_id, data_folder=data_folder)
else:

23
src/scgenerator/plotting.py
View File

@@ -7,7 +7,7 @@ from matplotlib.colors import ListedColormap
from scipy.interpolate import UnivariateSpline
from . import io, math
from .math import abs2, make_uniform_1D, span
from .math import abs2, length, make_uniform_1D, span
from .physics import pulse, units
from .defaults import default_plotting as defaults
@@ -296,7 +296,7 @@ def _finish_plot_2D(
folder_name = ""
if is_new_plot:
folder_name, file_name, fig, ax = io.plot_setup(
folder_name, file_name, fig, ax = plot_setup(
file_name=file_name, file_type=file_type, params=params
)
else:
@@ -549,11 +549,22 @@ def plot_results_2D(
[make_uniform_1D(v, x_axis, n=len(x_axis), method="linear") for v in values]
)
z = params["z_targets"]
lim_diff = 1e-5 * np.max(z)
dz_s = np.diff(z)
if not np.all(np.diff(dz_s) < lim_diff):
new_z = np.linspace(
*span(z), int(np.floor((np.max(z) - np.min(z)) / np.min(dz_s[dz_s > lim_diff])))
)
values = np.array(
[make_uniform_1D(v, z, n=len(new_z), method="linear") for v in values.T]
).T
z = new_z
return _finish_plot_2D(
values,
x_axis,
plt_range[2].label,
params["z_targets"],
z,
"propagation distance (m)",
log,
vmin,
@@ -681,7 +692,7 @@ def plot_results_1D(
folder_name = ""
if is_new_plot:
folder_name, file_name, fig, ax = io.plot_setup(
folder_name, file_name, fig, ax = plot_setup(
file_name=file_name, file_type=file_type, params=params
)
else:
@@ -845,11 +856,11 @@ def plot_avg(
if is_new_plot:
if add_coherence:
mode = "coherence_T" if transpose else "coherence"
folder_name, file_name, fig, (top, bot) = io.plot_setup(
folder_name, file_name, fig, (top, bot) = plot_setup(
file_name=file_name, file_type=file_type, params=params, mode=mode
)
else:
folder_name, file_name, fig, top = io.plot_setup(
folder_name, file_name, fig, top = plot_setup(
file_name=file_name, file_type=file_type, params=params
)
bot = top

2
src/scgenerator/spectra.py
View File

@@ -60,6 +60,8 @@ class Pulse(Sequence):
self.w_order = np.argsort(w)
self.w = w
self.wl = units.m.inv(self.w)
self.params["w"] = self.w
self.params["z_targets"] = self.z
def __iter__(self):
"""

12
src/scgenerator/utils.py
View File

@@ -5,12 +5,13 @@ scgenerator module but some function may be used in any python program
"""
import collections
import datetime as dt
import itertools
import logging
import re
import socket
from typing import Any, Callable, Iterator, List, Tuple, Union
from typing import Any, Callable, Iterator, List, Mapping, Tuple, Union
from asyncio import Event
import numpy as np
@@ -337,6 +338,15 @@ def parallelize(func, arg_iter, sim_jobs=4, progress_tracker_kwargs=None, const_
return np.array(results)
def deep_update(d: Mapping, u: Mapping):
for k, v in u.items():
if isinstance(v, collections.abc.Mapping):
d[k] = deep_update(d.get(k, {}), v)
else:
d[k] = v
return d
def formatted_hostname():
s = socket.gethostname().replace(".", "_")
return (PREFIX_KEY_BASE + s).upper()