Benchmark the reliability solvers on the problems

In this example, we show how to run all the methods on all the problems and get the computed probability.

import openturns as ot
import numpy as np
import otbenchmark as otb
import pandas as pd
from tqdm import tqdm
ot.Log.Show(ot.Log.NONE)

We import the list of reliability problems.

benchmarkProblemList = otb.ReliabilityBenchmarkProblemList()
numberOfProblems = len(benchmarkProblemList)
numberOfProblems
26

For each problem in the benchmark, print the problem name and the exact failure probability.

for i in range(numberOfProblems):
    problem = benchmarkProblemList[i]
    name = problem.getName()
    pf = problem.getProbability()
    print("#", i, " : ", name, ", exact PF : ", pf)
# 0  :  RP8 , exact PF :  0.0007897927545598118
# 1  :  RP14 , exact PF :  0.00077285
# 2  :  RP22 , exact PF :  0.004207305511299618
# 3  :  RP24 , exact PF :  0.00286
# 4  :  RP25 , exact PF :  4.148566293759747e-05
# 5  :  RP28 , exact PF :  1.4532945550025393e-07
# 6  :  RP31 , exact PF :  0.003226681209587691
# 7  :  RP33 , exact PF :  0.00257
# 8  :  RP35 , exact PF :  0.00347894632
# 9  :  RP38 , exact PF :  0.0081
# 10  :  RP53 , exact PF :  0.0313
# 11  :  RP55 , exact PF :  0.5600144282863704
# 12  :  RP54 , exact PF :  0.000998
# 13  :  RP57 , exact PF :  0.0284
# 14  :  RP75 , exact PF :  0.00981929872154689
# 15  :  RP89 , exact PF :  0.00543
# 16  :  RP107 , exact PF :  2.92e-07
# 17  :  RP110 , exact PF :  3.19e-05
# 18  :  RP111 , exact PF :  7.65e-07
# 19  :  RP63 , exact PF :  0.000379
# 20  :  RP91 , exact PF :  0.000697
# 21  :  RP60 , exact PF :  0.0456
# 22  :  RP77 , exact PF :  2.87e-07
# 23  :  Four-branch serial system , exact PF :  0.0022227950661944398
# 24  :  R-S , exact PF :  0.07864960352514257
# 25  :  Axial stressed beam , exact PF :  0.02919819462483095

Run several algorithms on a single problem

We want to run several algorithms on a single problem. We set the parameters of the algorithms and run them on a single problem.

maximumEvaluationNumber = 1000
maximumAbsoluteError = 1.0e-3
maximumRelativeError = 1.0e-3
maximumResidualError = 1.0e-3
maximumConstraintError = 1.0e-3
nearestPointAlgorithm = ot.AbdoRackwitz()
nearestPointAlgorithm.setMaximumCallsNumber(maximumEvaluationNumber)
nearestPointAlgorithm.setMaximumAbsoluteError(maximumAbsoluteError)
nearestPointAlgorithm.setMaximumRelativeError(maximumRelativeError)
nearestPointAlgorithm.setMaximumResidualError(maximumResidualError)
nearestPointAlgorithm.setMaximumConstraintError(maximumConstraintError)
i = 3
problem = benchmarkProblemList[i]
metaAlgorithm = otb.ReliabilityBenchmarkMetaAlgorithm(problem)

We try the FORM algorithm.

benchmarkFORM = metaAlgorithm.runFORM(nearestPointAlgorithm)
s1 = benchmarkFORM.summary()
print(s1)
computedProbability = 0.006209245091320783
exactProbability = 0.00286
absoluteError = 0.003349245091320783
numberOfCorrectDigits = 0.0
numberOfFunctionEvaluations = 6
numberOfDigitsPerEvaluation = 0.0

Then the SORM algorithm.

benchmarkSORM = metaAlgorithm.runSORM(nearestPointAlgorithm)
s2 = benchmarkSORM.summary()
print(s2)
computedProbability = 0.006209245091320783
exactProbability = 0.00286
absoluteError = 0.003349245091320783
numberOfCorrectDigits = 0.0
numberOfFunctionEvaluations = 6
numberOfDigitsPerEvaluation = 0.0
benchmarkMC = metaAlgorithm.runMonteCarlo(
    maximumOuterSampling=1000000, coefficientOfVariation=0.1, blockSize=1,
)
s3 = benchmarkMC.summary()
print(s3)
computedProbability = 0.0027376259307928274
exactProbability = 0.00286
absoluteError = 0.00012237406920717273
numberOfCorrectDigits = 1.3686766316046806
numberOfFunctionEvaluations = 36528
numberOfDigitsPerEvaluation = 3.7469246375511406e-05
benchmarkFORMIS = metaAlgorithm.runFORMImportanceSampling(
    nearestPointAlgorithm,
    maximumOuterSampling=1000,
    coefficientOfVariation=0.1,
    blockSize=1,
)
s4 = benchmarkFORMIS.summary()
print(s4)
computedProbability = 0.002782479204828666
exactProbability = 0.00286
absoluteError = 7.752079517133392e-05
numberOfCorrectDigits = 1.5669478142778805
numberOfFunctionEvaluations = 680
numberOfDigitsPerEvaluation = 0.0023043350209968833
benchmarkSS = metaAlgorithm.runSubsetSampling(
    maximumOuterSampling=5000, coefficientOfVariation=0.1, blockSize=1,
)
s5 = benchmarkSS.summary()
print(s5)
computedProbability = 0.0032215439999999933
exactProbability = 0.00286
absoluteError = 0.00036154399999999316
numberOfCorrectDigits = 0.8982048745292789
numberOfFunctionEvaluations = 15000
numberOfDigitsPerEvaluation = 5.988032496861859e-05

Run all algorithms on all problems and produce a single result table

For several algorithms and all the reliability problems, we want to estimate the failure probability and compare them.

We create a list of problem names.

problem_names = []
for i in range(numberOfProblems):
    problem = benchmarkProblemList[i]
    name = problem.getName()
    problem_names.append(name)
metrics = [
    "Exact",
    "FORM",
    "SORM",
    "Monte Carlo",
    "FORM-IS",
    "Subset",
]
results = np.zeros((numberOfProblems, len(metrics)))
maximumOuterSampling = 10 ** 2
blockSize = 10 ** 2
coefficientOfVariation = 0.0

for i in tqdm(range(numberOfProblems)):
    problem = benchmarkProblemList[i]
    results[i][0] = problem.getProbability()
    metaAlgorithm = otb.ReliabilityBenchmarkMetaAlgorithm(problem)
    benchmarkResult = metaAlgorithm.runFORM(nearestPointAlgorithm)
    results[i][1] = benchmarkResult.computedProbability
    benchmarkResult = metaAlgorithm.runSORM(nearestPointAlgorithm)
    results[i][2] = benchmarkResult.computedProbability
    benchmarkResult = metaAlgorithm.runMonteCarlo(
        maximumOuterSampling=maximumOuterSampling,
        coefficientOfVariation=coefficientOfVariation,
        blockSize=blockSize,
    )
    results[i][3] = benchmarkResult.computedProbability
    benchmarkResult = metaAlgorithm.runFORMImportanceSampling(
        nearestPointAlgorithm,
        maximumOuterSampling=maximumOuterSampling,
        coefficientOfVariation=coefficientOfVariation,
        blockSize=blockSize,
    )
    results[i][4] = benchmarkResult.computedProbability
    benchmarkResult = metaAlgorithm.runSubsetSampling(
        maximumOuterSampling=maximumOuterSampling,
        coefficientOfVariation=coefficientOfVariation,
        blockSize=blockSize,
    )
    results[i][5] = benchmarkResult.computedProbability

df = pd.DataFrame(results, index=problem_names, columns=metrics)
# df.to_csv("reliability_benchmark_table-output.csv")
df
  0%|          | 0/26 [00:00<?, ?it/s]
 12%|█▏        | 3/26 [00:00<00:00, 29.30it/s]
 27%|██▋       | 7/26 [00:00<00:00, 34.86it/s]
 42%|████▏     | 11/26 [00:00<00:00, 34.49it/s]
 62%|██████▏   | 16/26 [00:00<00:00, 38.58it/s]
 77%|███████▋  | 20/26 [00:01<00:00,  8.55it/s]
 88%|████████▊ | 23/26 [00:01<00:00, 10.51it/s]
100%|██████████| 26/26 [00:01<00:00, 14.89it/s]
Exact FORM SORM Monte Carlo FORM-IS Subset
RP8 7.897928e-04 6.598878e-04 7.838036e-04 0.0012 7.908163e-04 9.009000e-04
RP14 7.728500e-04 7.003011e-04 6.995436e-04 0.0010 7.876127e-04 8.275000e-04
RP22 4.207306e-03 6.209672e-03 4.390902e-03 0.0034 4.229721e-03 4.298000e-03
RP24 2.860000e-03 6.209245e-03 6.209245e-03 0.0030 2.838218e-03 2.776000e-03
RP25 4.148566e-05 0.000000e+00 0.000000e+00 0.0000 0.000000e+00 4.021906e-05
RP28 1.453295e-07 2.850470e-08 0.000000e+00 0.0000 1.451319e-07 1.764000e-07
RP31 3.226681e-03 2.275013e-02 2.275013e-02 0.0023 3.401714e-03 3.130000e-03
RP33 2.570000e-03 1.349898e-03 1.349898e-03 0.0028 2.243795e-03 2.656341e-03
RP35 3.478946e-03 1.349898e-03 2.134376e-03 0.0029 2.380555e-03 3.583000e-03
RP38 8.100000e-03 7.902212e-03 8.029356e-03 0.0081 7.896667e-03 7.209000e-03
RP53 3.130000e-02 1.180398e-01 2.986164e-02 0.0299 3.322343e-02 3.109000e-02
RP55 5.600144e-01 0.000000e+00 0.000000e+00 0.5539 0.000000e+00 5.632000e-01
RP54 9.980000e-04 5.555704e-02 3.554811e-03 0.0012 8.833106e-04 1.127871e-03
RP57 2.840000e-02 0.000000e+00 0.000000e+00 0.0287 0.000000e+00 2.881000e-02
RP75 9.819299e-03 0.000000e+00 0.000000e+00 0.0084 0.000000e+00 1.043000e-02
RP89 5.430000e-03 2.008594e-09 2.008594e-09 0.0047 3.479222e-05 5.224020e-03
RP107 2.920000e-07 2.866516e-07 2.866516e-07 0.0000 2.884108e-07 2.625101e-07
RP110 3.190000e-05 3.167124e-05 3.167124e-05 0.0000 3.147780e-05 2.413318e-05
RP111 7.650000e-07 0.000000e+00 0.000000e+00 0.0000 0.000000e+00 9.528314e-07
RP63 3.790000e-04 9.999966e-01 0.000000e+00 0.0003 0.000000e+00 3.653000e-04
RP91 6.970000e-04 6.994296e-04 7.011592e-04 0.0007 6.780302e-04 7.237000e-04
RP60 4.560000e-02 4.483968e-02 4.483968e-02 0.0465 4.473018e-02 4.461000e-02
RP77 2.870000e-07 0.000000e+00 0.000000e+00 0.0000 0.000000e+00 2.543238e-07
Four-branch serial system 2.222795e-03 0.000000e+00 0.000000e+00 0.0024 0.000000e+00 2.290410e-03
R-S 7.864960e-02 7.864960e-02 7.864960e-02 0.0806 7.958439e-02 8.150000e-02
Axial stressed beam 2.919819e-02 2.998280e-02 2.933256e-02 0.0305 2.817333e-02 2.819000e-02


Run several algorithms on all problems and get detailed statistics

Run several algorithms on all reliability benchmark problems: print statistics on each problem.

def FormatRow(benchmarkResult):
    """Format a single row of the benchmark table"""
    result = [
        benchmarkResult.exactProbability,
        benchmarkResult.computedProbability,
        benchmarkResult.absoluteError,
        benchmarkResult.numberOfCorrectDigits,
        benchmarkResult.numberOfFunctionEvaluations,
        benchmarkResult.numberOfDigitsPerEvaluation,
    ]
    return result
method_names = ["Monte-Carlo", "FORM", "SORM", "FORM-IS", "SUBSET"]

maximumOuterSampling = 10 ** 2
blockSize = 10 ** 2
coefficientOfVariation = 0.0

result = dict()
for i in range(numberOfProblems):
    problem = benchmarkProblemList[i]
    name = problem_names[i]
    exact_pf_name = "%10s" % ("Exact PF " + name[0:10])
    metrics = [
        exact_pf_name,
        "Estimated PF",
        "Absolute Error",
        "Correct Digits",
        "Function Calls",
        "Digits / Evaluation",
    ]
    results = np.zeros((len(method_names), len(metrics)))
    metaAlgorithm = otb.ReliabilityBenchmarkMetaAlgorithm(problem)
    # Monte-Carlo
    benchmarkResult = metaAlgorithm.runMonteCarlo(
        maximumOuterSampling=maximumOuterSampling,
        coefficientOfVariation=coefficientOfVariation,
        blockSize=blockSize,
    )
    results[0, :] = FormatRow(benchmarkResult)
    # FORM
    benchmarkResult = metaAlgorithm.runFORM(nearestPointAlgorithm)
    results[1, :] = FormatRow(benchmarkResult)
    # SORM
    benchmarkResult = metaAlgorithm.runSORM(nearestPointAlgorithm)
    results[2, :] = FormatRow(benchmarkResult)
    # FORM-IS
    benchmarkResult = metaAlgorithm.runFORMImportanceSampling(
        nearestPointAlgorithm,
        maximumOuterSampling=maximumOuterSampling,
        coefficientOfVariation=coefficientOfVariation,
        blockSize=blockSize,
    )
    results[3, :] = FormatRow(benchmarkResult)
    # Subset
    benchmarkResult = metaAlgorithm.runSubsetSampling(
        maximumOuterSampling=maximumOuterSampling,
        coefficientOfVariation=coefficientOfVariation,
        blockSize=blockSize,
    )
    results[4, :] = FormatRow(benchmarkResult)
    # Gather statistics and print them
    df = pd.DataFrame(results, index=method_names, columns=metrics,)
    # Format the columns for readability
    s = df.style.format(
        {
            exact_pf_name: lambda x: "{:.3e}".format(x),
            "Estimated PF": lambda x: "{:.3e}".format(x),
            "Absolute Error": lambda x: "{:.3e}".format(x),
            "Correct Digits": lambda x: "{:.1f}".format(x),
            "Function Calls": lambda x: "{:d}".format(int(x)),
            "Digits / Evaluation": lambda x: "{:.1f}".format(x),
        }
    )
    result[name] = s
result["RP33"]
  Exact PF RP33 Estimated PF Absolute Error Correct Digits Function Calls Digits / Evaluation
Monte-Carlo 2.570e-03 2.200e-03 3.700e-04 0.8 10000 0.0
FORM 2.570e-03 1.350e-03 1.220e-03 0.3 26 0.0
SORM 2.570e-03 1.350e-03 1.220e-03 0.3 51 0.0
FORM-IS 2.570e-03 2.341e-03 2.289e-04 1.1 10026 0.0
SUBSET 2.570e-03 2.559e-03 1.100e-05 2.4 30000 0.0


result["RP35"]
  Exact PF RP35 Estimated PF Absolute Error Correct Digits Function Calls Digits / Evaluation
Monte-Carlo 3.479e-03 4.200e-03 7.211e-04 0.7 10000 0.0
FORM 3.479e-03 1.350e-03 2.129e-03 0.2 20 0.0
SORM 3.479e-03 2.134e-03 1.345e-03 0.4 33 0.0
FORM-IS 3.479e-03 2.605e-03 8.735e-04 0.6 10020 0.0
SUBSET 3.479e-03 3.490e-03 1.105e-05 2.5 30000 0.0


Total running time of the script: (0 minutes 3.641 seconds)