Unity Scopes API
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Pages
Benchmark.h
1 /*
2  * Copyright (C) 2013 Canonical Ltd
3  *
4  * This program is free software: you can redistribute it and/or modify
5  * it under the terms of the GNU Lesser General Public License version 3 as
6  * published by the Free Software Foundation.
7  *
8  * This program is distributed in the hope that it will be useful,
9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11  * GNU Lesser General Public License for more details.
12  *
13  * You should have received a copy of the GNU Lesser General Public License
14  * along with this program. If not, see <http://www.gnu.org/licenses/>.
15  *
16  * Authored by: Thomas Voß <thomas.voss@canonical.com>
17  */
18 
19 #ifndef UNITY_SCOPES_TESTING_BENCHMARK_H
20 #define UNITY_SCOPES_TESTING_BENCHMARK_H
21 
22 #include <unity/scopes/ActionMetadata.h>
23 #include <unity/scopes/CannedQuery.h>
24 #include <unity/scopes/Result.h>
25 #include <unity/scopes/SearchMetadata.h>
26 
27 #include <unity/scopes/testing/Statistics.h>
28 
29 #include <chrono>
30 #include <functional>
31 #include <iosfwd>
32 #include <memory>
33 
34 namespace unity
35 {
36 
37 namespace scopes
38 {
39 class ScopeBase;
40 
41 namespace testing
42 {
43 
48 class Benchmark
49 {
50 public:
55  struct Result
56  {
58  std::size_t sample_size{0};
60  struct Timing : public Sample
61  {
63  typedef std::chrono::duration<double> Seconds;
64 
66  Timing() = default;
67  Timing(const Timing&) = default;
68  Timing(Timing&&) = default;
72  Sample::SizeType get_size() const;
78  void enumerate(const Sample::Enumerator& enumerator) const;
79 
88  bool is_significantly_faster_than_reference(const Timing& reference, double alpha = 0.05) const;
89 
100  double mean,
101  double std_dev,
102  double alpha = 0.05) const;
103 
112  bool is_significantly_slower_than_reference(const Timing& reference, double alpha = 0.05) const;
113 
124  double mean,
125  double std_dev,
126  double alpha = 0.05) const;
127 
129  Seconds min{Seconds::min()};
131  Seconds max{Seconds::min()};
133  Seconds mean{Seconds::min()};
135  Seconds std_dev{Seconds::min()};
137  Seconds kurtosis{Seconds::min()};
139  Seconds skewness{Seconds::min()};
141  std::vector<std::pair<Seconds, double>> histogram{};
143  std::vector<Seconds> sample{};
144  } timing{};
145 
151  void load_from(std::istream& in);
152 
158  void save_to(std::ostream& out);
159 
165  void load_from_xml(std::istream& in);
166 
172  void save_to_xml(std::ostream& out);
173  };
174 
180  {
182  std::size_t histogram_bin_count{10};
183  };
184 
190  {
192  std::size_t trial_count{25};
194  std::chrono::microseconds per_trial_timeout{std::chrono::seconds{10}};
197  };
198 
204  {
206  typedef std::function<
207  std::pair<
211 
219  };
220 
226  {
228  typedef std::function<
229  std::pair<
232  >()
234 
242  };
243 
249  {
251  typedef std::function<
252  std::pair<
256 
264  };
265 
271  {
273  typedef std::function<
274  std::tuple<
277  std::string,
278  std::string
279  >()
281 
289  };
290 
292  virtual ~Benchmark() = default;
293  Benchmark(const Benchmark&) = delete;
294  Benchmark(Benchmark&&) = delete;
295 
296  Benchmark& operator=(const Benchmark&) = delete;
297  Benchmark& operator=(Benchmark&&) = delete;
308  virtual Result for_query(const std::shared_ptr<unity::scopes::ScopeBase>& scope,
309  QueryConfiguration configuration) = 0;
310 
319  virtual Result for_preview(const std::shared_ptr<unity::scopes::ScopeBase>& scope,
320  PreviewConfiguration configuration) = 0;
321 
330  virtual Result for_activation(const std::shared_ptr<unity::scopes::ScopeBase>& scope,
331  ActivationConfiguration configuration) = 0;
332 
341  virtual Result for_action(const std::shared_ptr<unity::scopes::ScopeBase>& scope,
342  ActionConfiguration configuration) = 0;
343 
344 protected:
345  Benchmark() = default;
346 };
347 
348 bool operator==(const Benchmark::Result& lhs, const Benchmark::Result& rhs);
349 
350 std::ostream& operator<<(std::ostream&, const Benchmark::Result&);
351 
352 } // namespace testing
353 
354 } // namespace scopes
355 
356 } // namespace unity
357 
358 #endif
The ActionConfiguration struct constains all options controlling the benchmark of scope action activa...
Definition: Benchmark.h:270
std::function< std::pair< unity::scopes::Result, unity::scopes::ActionMetadata >) > Sampler
Definition: Benchmark.h:233
Seconds min
Definition: Benchmark.h:129
bool is_significantly_slower_than_reference(const Timing &reference, double alpha=0.05) const
Checks if a timing sample is statistically significantly slower than a reference timing sample...
Definition: Benchmark.cpp:216
bool is_significantly_faster_than_reference(const Timing &reference, double alpha=0.05) const
Checks if a timing sample is statistically significantly faster than a reference timing sample...
Definition: Benchmark.cpp:170
StatisticsConfiguration statistics_configuration
Definition: Benchmark.h:196
std::function< std::tuple< unity::scopes::Result, unity::scopes::ActionMetadata, std::string, std::string >) > Sampler
Definition: Benchmark.h:280
virtual Result for_action(const std::shared_ptr< unity::scopes::ScopeBase > &scope, ActionConfiguration configuration)=0
for_preview executes a benchmark to measure the scope's action activation performance.
The StatisticsConfiguration struct contains options controlling the calculation of benchmark result s...
Definition: Benchmark.h:179
void save_to_xml(std::ostream &out)
save_to_xml stores a result as xml to the given output stream.
Definition: Benchmark.cpp:113
Metadata passed with search requests.
Definition: SearchMetadata.h:41
TrialConfiguration trial_configuration
Definition: Benchmark.h:263
unity::scopes::testing::Benchmark::Result::Timing timing
Runtime-specific sample data.
virtual Result for_activation(const std::shared_ptr< unity::scopes::ScopeBase > &scope, ActivationConfiguration configuration)=0
for_preview executes a benchmark to measure the scope's activation performance.
std::function< void(ValueType)> Enumerator
Definition: Statistics.h:45
Metadata passed to scopes for preview and activation.
Definition: ActionMetadata.h:41
The attributes of a result returned by a Scope.
Definition: Result.h:51
The Sample class models the interface to a sample of raw observations and their statistical propertie...
Definition: Statistics.h:37
Seconds skewness
Definition: Benchmark.h:139
The PreviewConfiguration struct constains all options controlling the benchmark of scope preview oper...
Definition: Benchmark.h:225
TrialConfiguration trial_configuration
Definition: Benchmark.h:288
Seconds kurtosis
Definition: Benchmark.h:137
The ActivationConfiguration struct constains all options controlling the benchmark of scope activatio...
Definition: Benchmark.h:248
TrialConfiguration trial_configuration
Definition: Benchmark.h:218
void enumerate(const Sample::Enumerator &enumerator) const
Definition: Benchmark.cpp:164
std::size_t sample_size
Definition: Benchmark.h:58
std::function< std::pair< unity::scopes::Result, unity::scopes::ActionMetadata >) > Sampler
Definition: Benchmark.h:255
TrialConfiguration trial_configuration
Definition: Benchmark.h:241
void save_to(std::ostream &out)
save_to stores a result to the given output stream.
Definition: Benchmark.cpp:137
std::chrono::duration< double > Seconds
Definition: Benchmark.h:63
The Result struct encapsulates all of the result gathered from one individual benchmark run consistin...
Definition: Benchmark.h:55
std::vector< Seconds > sample
Definition: Benchmark.h:143
std::size_t SizeType
Definition: Statistics.h:41
std::function< std::pair< unity::scopes::CannedQuery, unity::scopes::SearchMetadata >) > Sampler
Definition: Benchmark.h:210
Sampler sampler
Definition: Benchmark.h:216
std::size_t histogram_bin_count
Definition: Benchmark.h:182
Seconds std_dev
Definition: Benchmark.h:135
std::size_t trial_count
Definition: Benchmark.h:192
virtual Result for_preview(const std::shared_ptr< unity::scopes::ScopeBase > &scope, PreviewConfiguration configuration)=0
for_preview executes a benchmark to measure the scope's preview performance.
Sampler sampler
Definition: Benchmark.h:286
Parameters of a search query.
Definition: CannedQuery.h:46
Sample::ValueType get_variance() const
Definition: Benchmark.cpp:159
std::vector< std::pair< Seconds, double > > histogram
Definition: Benchmark.h:141
Sample::ValueType get_mean() const
Definition: Benchmark.cpp:154
Seconds max
Definition: Benchmark.h:131
virtual Result for_query(const std::shared_ptr< unity::scopes::ScopeBase > &scope, QueryConfiguration configuration)=0
for_query executes a benchmark to measure the scope's query performance.
The QueryConfiguration struct constains all options controlling the benchmark of scope query operatio...
Definition: Benchmark.h:203
The TrialConfiguration struct contains options controlling the execution of individual trials...
Definition: Benchmark.h:189
Sample::SizeType get_size() const
Definition: Benchmark.cpp:149
The Benchmark class defines an interface to provide scope authors with runtime benchmarking capabilit...
Definition: Benchmark.h:48
Seconds mean
Definition: Benchmark.h:133
void load_from(std::istream &in)
load_from restores a result from the given input stream.
Definition: Benchmark.cpp:125
std::chrono::microseconds per_trial_timeout
Definition: Benchmark.h:194
void load_from_xml(std::istream &in)
load_from_xml restores a result stored as xml from the given input stream.
Definition: Benchmark.cpp:101
double ValueType
Definition: Statistics.h:43