Merge branch 'release-2.0.0b0'

.gitignore

@@ -52,3 +52,10 @@ docs/_build/
 
 # PyBuilder
 target/
+
+# Virtualenv
+virt/
+virt3/
+
+# PyCharm
+.idea/

CHANGES.txt

@@ -0,0 +1,31 @@
+PyPWA 2.0.0-Alpha.0:
+-Boasts a threaded GeneralShell with relatively no bugs in it.
+-A functioning python packaging system so that the General Shell can be installed with ease.
+-A Configuration file vs FnTemplate so that users can use the same GeneralShell for all fits.
+
+Complete changelog since PyPWA 1.1:
+
+* c27319a : Fixed a typo in the generate_config method of YamlConfig
+* 7ee371a : Updated setup.py to the current version and status of PyPWA, updated setup.cfg to point to README.md instead of README.txt, and updated MANIFEST.in to contain root licenese, changes, and readme files.
+* 80d27ab : Fixed a series of typos in YamlConfig and DataCalc that prevented GeneralFitting from executing properly.
+* 2089c9b : Added ignore for PyCharm files
+* edc7939 : Fixed bug #9 and enhancement #7. Fixed some bugs that prevented running, and converted private methods and variables to public.
+* fe46418 : Added documentation for undocumented methods and functions, also improved documenation for some methods.
+* 431c201 : Removed General Shell from limbo since the threaded general shell is part of the package now.
+* cfabc1a : Remove redundant LICENSE and TODO file
+* c87f120 : Updated readme to include the state of PyPWA and give a basic example of how to use the GeneralFitting
+* 5354d72 : Updated examples to better explain how to propperly use the program.
+*   57731cb : Merge branch 'threading-generalShell-resourceImprovement' into development for improved threading.
+|\  
+| *   193e082 : Merge branch 'development' into threading-generalShell-resourceImprovement
+| |\  
+| |/  
+|/|   
+* | 760f633 : Fixed setup.py pointing to README.txt over README.md
+| * 232d2a5 : Fixed a bug where "Number of threads" was used instead of "Number of Threads". Fixed array splitting.
+| * c62f23a : Modified how threads are handled to improve overall performance within the General Shell. Performance increase should be more linear now with the addition of more threads.
+|/  
+* 5420529 : Removed unintended virtualenv files
+* 9be3ec7 : Modified .gitignore to ignore virtualenv files.
+* f417348 : Merged in work on threaded GeneralShell with setuptools packaging utils.
+* 1773d3c : Removed deprecitated and outdated files.

LICENSE → LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2014 Jefferson Lab
+Copyright (c) <year> <copyright holders>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -9,14 +9,13 @@ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 copies of the Software, and to permit persons to whom the Software is
 furnished to do so, subject to the following conditions:
 
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
 
 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

MANIFEST.in

@@ -0,0 +1,3 @@
+include README.md
+include LICENSE.txt
+include CHANGES.txt

PyPWA/__init__.py

@@ -0,0 +1,20 @@
+"""
+PyPWA is an attempt to have a set multiprocessing tools to easy the
+Partial Wave Analysis process.
+
+Current there is a multiprocessing tool for both the Acceptance
+Rejection Model, and the Maximum-Likelihood Estimation for fitting.
+
+The tools work with Kinematic Variables defined in standard text files,
+Comma Seperated Variables, and Tab Seperated Variables.
+
+Example:
+    For both the GeneralFitting and the GeneralSimulator you should
+    first run [tool] -wc, ie:
+        $ GeneralFitting -wc
+    then afterwords modify and rename the Example.yml and Example.py
+    files that are generated.
+    Lastly to actually run the tool with your freshly created
+    configuration file, run [tool] <configuration>.yml, ie:
+        $ GeneralFitting Example.yml
+"""

PyPWA/core/__init__.py

@@ -0,0 +1,7 @@
+"""
+This module is used to actually processes loaded data.
+
+There entire module is based off of multiprocessing, having objects
+bassed off of the Process module and Pipe module for effective
+communication.
+"""

PyPWA/core/console_main.py

@@ -0,0 +1,177 @@
+"""
+Main objects for console PyPWA tools
+"""
+__author__ = "Mark Jones"
+__credits__ = ["Mark Jones", "Josh Pond"]
+__license__ = "MIT"
+__version__ = "2.0.0"
+__maintainer__ = "Mark Jones"
+__email__ = "maj@jlab.org"
+__status__ = "Beta0"
+
+import PyPWA.data.file_manager, PyPWA.proc.calculation_tools, PyPWA.proc.calculation
+
+class Fitting(object):
+    """Main General Fitting Object
+    Args:
+        config (dict): A dictionary with all the configuration packed into it.
+        cwd (str): The current working directory for the application
+    """
+
+    def __init__(self, config, cwd):
+        self.generated_length = config["Likelihood Information"]["Generated Length"]
+        self.function_location = config["Likelihood Information"]["Function's Location"]
+        self.amplitude_name = config["Likelihood Information"]["Processing Name"]
+        self.setup_name = config["Likelihood Information"]["Setup Name"]
+        self.data_location = config["Data Information"]["Data Location"]
+        self.accepted_location = config["Data Information"]["Accepted Monte Carlo Location"]
+        self.qfactor_location = config["Data Information"]["QFactor List Location"]
+        self.initial_settings = config["Minuit's Settings"]["Minuit's Initial Settings"]
+        self.parameters = config["Minuit's Settings"]["Minuit's Parameters"]
+        self.strategy = config["Minuit's Settings"]["Minuit's Strategy"]
+        self.set_up = config["Minuit's Settings"]["Minuit's Set Up"]
+        self.ncall = config["Minuit's Settings"]["Minuit's ncall"]
+        self.num_threads = config["General Settings"]["Number of Threads"]
+        self.use_qfactor = config["Generated Settings"]["Use QFactor"]
+        self.cwd = cwd
+
+
+    def start(self):
+        """Starts fitting process"""
+
+        print("Parsing files into memory.\n")
+        parse = PyPWA.data.file_manager.MemoryInterface()
+        data = parse.parse(self.data_location)
+        accepted = parse.parse(self.accepted_location)
+        if self.use_qfactor:
+            qfactor = parse.parse(self.qfactor_location)
+        else:
+            qfactor = numpy.ones(shape=len(data.values()[0]))
+
+        print("Loading users function.\n")
+        functions = PyPWA.proc.calculation_tools.FunctionLoading(self.cwd, self.function_location, self.amplitude_name, self.setup_name)
+        amplitude_function = functions.return_amplitude()
+        setup_function = functions.return_setup()
+
+        calc = PyPWA.proc.calculation.MaximumLogLikelihoodEstimation(self.num_threads, self.parameters, data, accepted, qfactor, self.generated_length, amplitude_function, setup_function)
+        minimalization = PyPWA.proc.calculation_tools.Minimalizer(calc.run, self.parameters, self.initial_settings, self.strategy, self.set_up, self.ncall)
+
+        print("Starting minimalization.\n")
+        minimalization.min()
+        calc.stop()
+
+
+class Simulator(object):
+    """Main General Simulator Object
+    Args:
+        config (dict): A dictionary with all the configuration packed into it.
+        cwd (str): The current working directory for the application
+    """
+
+    def __init__(self, config, cwd):
+        self.function_location = config["Simulator Information"]["Function's Location"]
+        self.amplitude_name = config["Simulator Information"]["Processing Name"]
+        self.setup_name = config["Simulator Information"]["Setup Name"]
+        self.parameters = config["Simulator Information"]["Parameters"]
+        self.num_threads = config["Simulator Information"]["Number of Threads"]
+        self.data_location = config["Data Information"]["Monte Carlo Location"]
+        self.save_location = config["Data Information"]["Save Location"]
+        self.cwd = cwd
+
+
+    def start(self):
+        """Starts Rejection"""
+
+        print("Parsing data into memory.\n")
+        data_manager = PyPWA.data.file_manager.MemoryInterface()
+        data = data_manager.parse(self.data_location)
+
+        print("Loading users functions.\n")
+        functions = PyPWA.proc.calculation_tools.FunctionLoading(self.cwd, self.function_location, self.amplitude_name, self.setup_name )
+        amplitude_function = functions.return_amplitude()
+        setup_function = functions.return_setup()
+
+        print("Running Simulation")
+        rejection_method = PyPWA.proc.calculation.AcceptanceRejctionMethod( self.num_threads, data, amplitude_function, setup_function, self.parameters)
+
+        rejection_list = rejection_method.run()
+
+        print("Saving Data")
+        data_manager.write(self.save_location, rejection_list )
+
+
+class Configurations(object):
+    """Static class that returns the example txt"""
+
+    @staticmethod
+    def fitting_config():
+        """
+        Retruns:
+            str: Example.yml for GeneralFitting
+        """
+        return """\
+Likelihood Information: #There must be a space bewteen the colon and the data
+    Generated Length : 10000   #Number of Generated events
+    Function's Location : Example.py   #The python file that has the functions in it
+    Processing Name : the_function  #The name of the processing function
+    Setup Name :  the_setup   #The name of the setup function, called only once before fitting
+Data Information:
+    Data Location : /home/user/foobar/data.txt #The location of the data
+    Accepted Monte Carlo Location: /home/foobar/fit/AccMonCar.txt   #The location of the Accepted Monte Carlo
+    QFactor List Location : /home/foobar/fit/Qfactor.txt #The location of the Qfactors
+Minuit's Settings:
+    Minuit's Initial Settings : { A1: 1, limit_A1: [0, 2500], # You can arrange this value however you would like as long as the each line ends in either a "," or a "}"
+        A2: 2, limit_A2: [-2,3],
+        A3: 0.1, A4: -10,
+        A5: -0.00001 }  #Iminuit settings in a single line
+    Minuit's Parameters: [ A1, A2, A3, A4, A5 ]   #The name of the Parameters passed to Minuit
+    Minuit's Strategy : 1
+    Minuit's Set Up: 0.5
+    Minuit's ncall: 1000
+General Settings:
+    Number of Threads: 1   #Number of threads to use. Set to one for debug
+    Use QFactor: True   #Boolean, using Qfactor or not
+"""
+
+
+    @staticmethod
+    def simulator_config():
+        """
+        Returns:
+            str: Example.yml for GeneralSimulator
+        """
+        return """\
+Simulator Information: #There must be a space bewteen the colon and the data
+    Function's Location : Example.py   #The python file that has the functions in it
+    Processing Name : the_function  #The name of the processing function
+    Setup Name :  the_setup   #The name of the setup function, called only once before fitting
+    Parameters : { A1: 1, A2: 2, A3: 0.1, A4: -10, A5: -0.00001 }
+    Number of Threads: 2
+Data Information:
+    Monte Carlo Location : /home/user/foobar/data.txt #The location of the data
+    Save Location : /home/user/foobabar/weights.txt #Where you want to save the weights
+    """
+
+
+    @staticmethod
+    def example_function():
+        """
+        Returns:
+            str: Example.py for both GeneralShell tools.
+        """
+        return """\
+import numpy
+
+def the_function(the_array, the_params): #You can change both the variable names and function name
+    the_size = len(the_array.values()[0]) #You can change the variable name here, or set the length of values by hand
+    values = numpy.zeros(shape=the_size)
+    for x in range(the_size):
+        #Here is where you define your function.
+        #Your array has to have a [x] after it so the for loop can iterate through all the events in the array
+        values[x] = the_param["A1"] + the_array["kvar"][x]
+    return values
+
+def the_setup(): #This function can be renamed, but will not be sent any arguments.
+    #This function will be ran once before the data is Minuit begins.
+    pass
+"""

PyPWA/data/__init__.py

@@ -0,0 +1,16 @@
+"""
+This module loads data from various data types to be used
+inside the program as they would like. Data types supported
+are the classic Kinematic Variable files defined by Josh
+Pond, the QFactors List, the Weight list, the Condensed
+single line Weight, and the Tab or Comma seperated
+kinematic variables.
+
+Examples:
+    To load data from file:
+        file = PyPWA.data.file_manager.MemoryInterface()
+        file.parse(path_to_file)
+    To write data to file:
+        file = PyPWA.data.file_manager.MemoryInterface()
+        file.write(path_to_file, the_data)
+"""

PyPWA/data/cache.py

@@ -0,0 +1,86 @@
+"""
+Cache objects for data module are stored here.
+"""
+__author__ = "Mark Jones"
+__credits__ = ["Mark Jones"]
+__license__ = "MIT"
+__version__ = "2.0.0"
+__maintainer__ = "Mark Jones"
+__email__ = "maj@jlab.org"
+__status__ = "Beta0"
+
+import pickle, hashlib, os
+
+class StandardCache(object):
+    """
+    This is a horribly written object, please ignore. It currently is unimplemented.
+
+    Todo:
+        Write a propper init, imply
+    """
+
+    def __init__(self):
+        raise NotImplemented
+
+
+    def find_hash(self, the_file):
+        """ Iterates over file and returns the hash
+        Args:
+            the_file (str): the file you want to be loaded.
+        Returns:
+            str: sha256 hash of file
+        """
+        try:
+            with open(the_file, "r") as a_file:
+                for line in a_file.readlines():
+                    line = line.strip("\n")
+                    the_hash = hashlib.sha256(line)
+        except IOError:
+            raise AttributeError(the_file + " doesn't exsist. Please check your configuration and try again.")
+        return the_hash.hexdigest()
+
+
+    def cache_location(self, file_location):
+        """ Determines where to store cache
+        Args:
+            file_location (str): the location of the original file.
+        Returns:
+            str: Absolute location of where to store the cache
+        """
+        the_path = file_location.split("/")
+        path_length = len(the_path) - 1
+        file_name = the_path[path_length]
+        file_name = os.path.splitext(file_name)[0]
+        file_name = "." + file_name + ".pickle-cache"
+        the_path[path_length] = file_name
+        return "/".join(the_path)
+
+
+    def make_cache(self, data, the_file):
+        """Pickles the data to file.
+        Args:
+            data: Anything that needs to be cached
+            the_file (str): Where to write the cache
+        """
+        with open(the_file, "wb") as a_file:
+            pickle.dump(data, a_file, protocol=pickle.HIGHEST_PROTOCOL)
+
+
+    def load_cache(self, the_file):
+        """Attempts to load the cache from file,
+        Args:
+            the_file (str): location of the cache
+        Returns:
+            object: Whatever was stored in the cache
+        """
+        try:
+            with open(the_file, "r")  as a_file:
+                cache = pickle.load(a_file)
+            self.cache_success = True
+            return cache
+        except EOFError:
+            self.cache_success = False
+            return {"files_hash":0}
+        except IOError:
+            self.cache_success = False
+            return {"files_hash":0}