Convergence of estimators on Ishigami

In this example, we present the convergence of the sensitivity indices of the Ishigami test function.

We compare different estimators.

  • Sampling methods with different estimators: Saltelli, Mauntz-Kucherenko, Martinez, Jansen,

  • Sampling methods with different design of experiments: Monte-Carlo, LHS, Quasi-Monte-Carlo,

  • Polynomial chaos.

import openturns as ot
import otbenchmark as otb
import openturns.viewer as otv
import numpy as np
maximumElapsedTime = 0.5

When we estimate Sobol’ indices, we may encounter the following warning messages: ` WRN - The estimated first order Sobol index (2) is greater than its total order index... WRN - The estimated total order Sobol index (2) is lesser than first order index ... ` Lots of these messages are printed in the current Notebook. This is why we disable them with:

ot.Log.Show(ot.Log.NONE)
problem = otb.IshigamiSensitivity()
print(problem)
name = Ishigami
distribution = JointDistribution(Uniform(a = -3.14159, b = 3.14159), Uniform(a = -3.14159, b = 3.14159), Uniform(a = -3.14159, b = 3.14159), IndependentCopula(dimension = 3))
function = ParametricEvaluation([X1,X2,X3,a,b]->[sin(X1) + a * sin(X2)^2 + b * X3^4 * sin(X1)], parameters positions=[3,4], parameters=[a : 7, b : 0.1], input positions=[0,1,2])
firstOrderIndices = [0.313905,0.442411,0]
totalOrderIndices = [0.557589,0.442411,0.243684]
distribution = problem.getInputDistribution()
model = problem.getFunction()

Exact first and total order

exactFirstOrder = problem.getFirstOrderIndices()
print(exactFirstOrder)
exactTotalOrder = problem.getTotalOrderIndices()
print(exactTotalOrder)
[0.313905,0.442411,0]
[0.557589,0.442411,0.243684]

Perform sensitivity analysis

Create X/Y data

ot.RandomGenerator.SetSeed(0)
size = 10000
inputDesign = ot.SobolIndicesExperiment(distribution, size).generate()
outputDesign = model(inputDesign)

Compute first order indices using the Saltelli estimator

sensitivityAnalysis = ot.SaltelliSensitivityAlgorithm(inputDesign, outputDesign, size)
computedFirstOrder = sensitivityAnalysis.getFirstOrderIndices()
computedTotalOrder = sensitivityAnalysis.getTotalOrderIndices()

Compare with exact results

print("Sample size : ", size)
# First order
# Compute absolute error (the LRE cannot be computed,
# because S can be zero)
print("Computed first order = ", computedFirstOrder)
print("Exact first order    = ", exactFirstOrder)
# Total order
print("Computed total order = ", computedTotalOrder)
print("Exact total order    = ", exactTotalOrder)
Sample size :  10000
Computed first order =  [0.302745,0.460846,0.0066916]
Exact first order    =  [0.313905,0.442411,0]
Computed total order =  [0.574996,0.427126,0.256689]
Exact total order    =  [0.557589,0.442411,0.243684]
dimension = distribution.getDimension()

Compute componentwise absolute error.

firstOrderAE = ot.Point(np.abs(exactFirstOrder - computedFirstOrder))
totalOrderAE = ot.Point(np.abs(exactTotalOrder - computedTotalOrder))
print("Absolute error")
for i in range(dimension):
    print(
        f"AE(S{i}) = {firstOrderAE[i]:.4f}, AE(T{i}) = {totalOrderAE[i]:.4f}"
    )
Absolute error
AE(S0) = 0.0112, AE(T0) = 0.0174
AE(S1) = 0.0184, AE(T1) = 0.0153
AE(S2) = 0.0067, AE(T2) = 0.0130
metaSAAlgorithm = otb.SensitivityBenchmarkMetaAlgorithm(problem)
for estimator in ["Saltelli", "Martinez", "Jansen", "MauntzKucherenko", "Janon"]:
    print("Estimator:", estimator)
    benchmark = otb.SensitivityConvergence(
        problem,
        metaSAAlgorithm,
        numberOfRepetitions=4,
        maximumElapsedTime=maximumElapsedTime,
        sampleSizeInitial=20,
        estimator=estimator,
    )
    grid = benchmark.plotConvergenceGrid(verbose=False)
    view = otv.View(grid)
    figure = view.getFigure()
    _ = figure.suptitle(f"{problem.getName()}, {estimator}")
    figure.set_figwidth(10.0)
    figure.set_figheight(5.0)
    figure.subplots_adjust(wspace=0.4, hspace=0.4)
  • Ishigami, Saltelli
  • Ishigami, Martinez
  • Ishigami, Jansen
  • Ishigami, MauntzKucherenko
  • Ishigami, Janon
Estimator: Saltelli
Estimator: Martinez
Estimator: Jansen
Estimator: MauntzKucherenko
Estimator: Janon
benchmark = otb.SensitivityConvergence(
    problem,
    metaSAAlgorithm,
    numberOfRepetitions=4,
    maximumElapsedTime=maximumElapsedTime,
    sampleSizeInitial=20,
    estimator="Saltelli",
    samplingMethod="MonteCarlo",
)
graph = benchmark.plotConvergenceCurve()
graph.setLegendPosition("upper left")
graph.setLegendCorner((1.0, 1.0))
_ = otv.View(graph, figure_kw={"figsize": (4.0, 3.0)})
Ishigami, Saltelli, MonteCarlo
grid = ot.GridLayout(1, 3)
maximumAbsoluteError = 1.0
minimumAbsoluteError = 1.0e-5
samplingMethodList = ["MonteCarlo", "LHS", "QMC"]
estimator = "Saltelli"
for samplingMethodIndex, samplingMethod in enumerate(samplingMethodList):
    samplingMethod = samplingMethodList[samplingMethodIndex]
    benchmark = otb.SensitivityConvergence(
        problem,
        metaSAAlgorithm,
        numberOfRepetitions=4,
        maximumElapsedTime=maximumElapsedTime,
        sampleSizeInitial=20,
        estimator=estimator,
        samplingMethod=samplingMethod,
    )
    graph = benchmark.plotConvergenceCurve()
    # Change bounding box
    box = graph.getBoundingBox()
    bound = box.getLowerBound()
    bound[1] = minimumAbsoluteError
    box.setLowerBound(bound)
    bound = box.getUpperBound()
    bound[1] = maximumAbsoluteError
    box.setUpperBound(bound)
    graph.setBoundingBox(box)
    if samplingMethodIndex < len(samplingMethodList) - 1:
        graph.setLegends([""])
    else:
        graph.setLegendPosition("upper left")
        graph.setLegendCorner((1.0, 1.0))
    if samplingMethodIndex > 0:
        graph.setYTitle("")
    graph.setTitle(f"{samplingMethod}")
    grid.setGraph(0, samplingMethodIndex, graph)
grid.setTitle(f"Ishigami, {estimator}")
_ = otv.View(grid, figure_kw={"figsize": (8.0, 4.0)})
Ishigami, Saltelli, MonteCarlo, LHS, QMC

Use polynomial chaos.

sparse = True
benchmark = otb.SensitivityConvergence(
    problem,
    metaSAAlgorithm,
    numberOfExperiments=12,
    numberOfRepetitions=1,
    maximumElapsedTime=5.0,
    sampleSizeInitial=80,
    useSampling=False,
    totalDegree=8,
    hyperbolicQuasiNorm=1.0,
    sampleSizeFactor=1.5,
    sparse=sparse,
)
graph = benchmark.plotConvergenceCurve(verbose=True)
graph.setLegendPosition("upper left")
graph.setLogScale(ot.GraphImplementation.LOGX)
graph.setLegendCorner((1.0, 1.0))
_ = otv.View(graph, figure_kw={"figsize": (4.0, 3.0)})
Ishigami, P.C., Degree=8
Elapsed = 0.0 (s), Sample size = 80
Error in experiment 0, repetition 0, sample size 80: The number of candidate coefficients is 165 is larger or equal to the sample size 80
Elapsed = 0.0 (s), Sample size = 120
Error in experiment 1, repetition 0, sample size 120: The number of candidate coefficients is 165 is larger or equal to the sample size 120
Elapsed = 0.0 (s), Sample size = 180
Elapsed = 0.0 (s), Sample size = 270
Elapsed = 0.1 (s), Sample size = 405
Elapsed = 0.3 (s), Sample size = 608
Elapsed = 0.6 (s), Sample size = 912
Elapsed = 1.6 (s), Sample size = 1368
Elapsed = 3.8 (s), Sample size = 2052
Elapsed = 7.3 (s) > 5.0 (s), stopping the simulation.
Elapsed = 7.33 (s)
otv.View.ShowAll()

Total running time of the script: (0 minutes 17.804 seconds)