Collect Result Information — collectResult • mosmafs

Merge both log and log.newinds data for a complete data.frame with information about progress (both on training data and on holdout data) and ressource usage.

collectResult(
  ecr.object,
  aggregate.perresult = list(domHV = function(x) computeHV(x, ref.point)),
  aggregate.perobjective = list("min", "mean", "max"),
  ref.point = smoof::getRefPoint(ecr.object$control$task$fitness.fun),
  cor.fun = cor
)

Arguments

ecr.object	`[MosmafsResult]` `slickEcr()` result to analyse.
aggregate.perresult	`[list]` list of functions to apply to fitness and holdout fitness. Every entry must either be a `character(1)` naming the function to use, or a function, in which that entry must have a name. Each function must return exactly one numeric value when fed a fitness matrix of one generation. This is ignored for single-objective runs.
aggregate.perobjective	`[list]` list of functions to apply to fitness and holdout fitness matrix rows, formatted like `aggregate.perresult`. Each function must return exactly one numeric value when fed a fitness vector.
ref.point	`[numeric]` reference point to use for HV computation.
cor.fun	`[function]` function to use for calculation of correlation between objective and holdout. Must take two `numeric` arguments and return a `numeric(1)`.

Value

data.frame

Examples

library(mlrCPO)
#> Loading required package: mlr
#> 'mlr' is in maintenance mode since July 2019. Future development
#> efforts will go into its successor 'mlr3' (<https://mlr3.mlr-org.com>).

# Setup of optimization problem 
ps.simple <- pSS(
 a: numeric [0, 10],
 selector.selection: logical^10)

mutator.simple <- combine.operators(ps.simple,
 a = mutGauss,
 selector.selection = mutBitflipCHW)

crossover.simple <- combine.operators(ps.simple,
 a = recSBX,
 selector.selection = recPCrossover)

initials <- sampleValues(ps.simple, 30, discrete.names = TRUE)

fitness.fun <- smoof::makeMultiObjectiveFunction(
 sprintf("simple test"),
 has.simple.signature = FALSE, par.set = ps.simple, n.objectives = 2,
 noisy = TRUE,
 ref.point = c(10, 1),
 fn = function(args, fidelity = NULL, holdout = FALSE) {
   pfeat <- mean(args$selector.selection)
   c(perform = args$a, pfeat = pfeat)
 })
fitness.fun.single <- smoof::makeMultiObjectiveFunction(
 sprintf("simple test"),
 has.simple.signature = FALSE, par.set = ps.simple, n.objectives = 1,
 noisy = TRUE,
 ref.point = c(10),
 fn = function(args, fidelity = NULL, holdout = FALSE) {
   propfeat <- mean(args$selector.selection)
   c(propfeat = propfeat)
 })

# Run NSGA-II
results <- slickEcr(fitness.fun = fitness.fun, lambda = 10, population = initials,
 mutator = mutator.simple, recombinator = crossover.simple, generations = 10)

# Collect results
colres <- collectResult(results)
print(colres)
#>    gen runtime evals eval.perform.min eval.perform.mean eval.perform.max
#> 1    0   0.030    30        0.8075014          5.401989         9.951230
#> 2    1   0.038    40        0.7232842          4.765943         9.477639
#> 3    2   0.047    50        0.7232842          3.966252         9.510273
#> 4    3   0.058    60        0.7232842          3.550201         9.561445
#> 5    4   0.068    70        0.5644004          3.546856         9.561445
#> 6    5   0.080    80        0.5032208          3.010343         9.477639
#> 7    6   0.092    90        0.5032208          2.376331         6.897384
#> 8    7   0.105   100        0.5032208          2.052842         7.762602
#> 9    8   0.119   110        0.5032208          1.591523         7.762602
#> 10   9   0.126   120        0.4561178          1.568028         7.762602
#> 11  10   0.135   130        0.4561178          1.361632         7.762602
#>    eval.pfeat.min eval.pfeat.mean eval.pfeat.max eval.domHV hout.perform.min
#> 1             0.2       0.5066667            1.0   6.342119        0.8075014
#> 2             0.2       0.4766667            1.0   6.375806        0.7232842
#> 3             0.2       0.4500000            1.0   6.375806        0.7232842
#> 4             0.2       0.4166667            0.7   6.792824        0.7232842
#> 5             0.2       0.3833333            0.7   7.399575        0.5644004
#> 6             0.2       0.3733333            0.7   7.433930        0.5032208
#> 7             0.2       0.3733333            0.7   7.435411        0.5032208
#> 8             0.1       0.3600000            0.6   7.688097        0.5032208
#> 9             0.1       0.3666667            0.6   8.399393        0.5032208
#> 10            0.1       0.3466667            0.6   8.438343        0.4561178
#> 11            0.1       0.3033333            0.6   8.576795        0.4561178
#>    hout.perform.mean hout.perform.max hout.pfeat.min hout.pfeat.mean
#> 1           5.401989         9.951230            0.2       0.5066667
#> 2           4.765943         9.477639            0.2       0.4766667
#> 3           3.966252         9.510273            0.2       0.4500000
#> 4           3.550201         9.561445            0.2       0.4166667
#> 5           3.546856         9.561445            0.2       0.3833333
#> 6           3.010343         9.477639            0.2       0.3733333
#> 7           2.376331         6.897384            0.2       0.3733333
#> 8           2.052842         7.762602            0.1       0.3600000
#> 9           1.591523         7.762602            0.1       0.3666667
#> 10          1.568028         7.762602            0.1       0.3466667
#> 11          1.361632         7.762602            0.1       0.3033333
#>    hout.pfeat.max hout.domHV true.hout.domHV naive.hout.domHV cor.perform
#> 1             1.0   6.342119        6.342119         6.342119           1
#> 2             1.0   6.375806        6.375806         6.375806           1
#> 3             1.0   6.375806        6.375806         6.375806           1
#> 4             0.7   6.792824        6.792824         6.792824           1
#> 5             0.7   7.399575        7.399575         7.399575           1
#> 6             0.7   7.433930        7.433930         7.433930           1
#> 7             0.7   7.435411        7.435411         7.435411           1
#> 8             0.6   7.688097        7.688097         7.688097           1
#> 9             0.6   8.399393        8.399393         8.399393           1
#> 10            0.6   8.438343        8.438343         8.438343           1
#> 11            0.6   8.576795        8.576795         8.576795           1
#>    cor.pfeat
#> 1          1
#> 2          1
#> 3          1
#> 4          1
#> 5          1
#> 6          1
#> 7          1
#> 8          1
#> 9          1
#> 10         1
#> 11         1