From 7ae3973348da46f10b03ff1c5b90bc31a49d8964 Mon Sep 17 00:00:00 2001 From: daijian Date: Wed, 27 Jan 2016 10:40:46 +0800 Subject: [PATCH 1/3] add clocks for forward and backward functions. --- include/singa/utils/clock.h | 48 +++++++++++++++++++++++++++++ src/utils/clock.cc | 61 +++++++++++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+) create mode 100644 include/singa/utils/clock.h create mode 100644 src/utils/clock.cc diff --git a/include/singa/utils/clock.h b/include/singa/utils/clock.h new file mode 100644 index 0000000000..785f48f9cf --- /dev/null +++ b/include/singa/utils/clock.h @@ -0,0 +1,48 @@ +/************************************************************ +* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +* +*************************************************************/ + +#ifndef SINGA_CLOCK_H_ +#define SINGA_CLOCK_H_ + +#include + +namespace singa { + +class Clock { + public: + void start(); + + double end(); + double end(int iteration_num, std::string content); + void endWithLog(std::string content); + void endWithLog(int iteration_num, std::string content); + + protected: + int start_t = -1, end_t = -1; +}; + + +} + + + + +#endif diff --git a/src/utils/clock.cc b/src/utils/clock.cc new file mode 100644 index 0000000000..b14f11fcb7 --- /dev/null +++ b/src/utils/clock.cc @@ -0,0 +1,61 @@ +/************************************************************ +* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +* +*************************************************************/ + +#include "singa/utils/clock.h" + +#include +#include +#include +#include "singa/utils/cluster.h" +#include "singa/utils/factory.h" +#include "singa/utils/singleton.h" +#include "singa/utils/context.h" + +namespace singa { + +void Clock::start() { + start_t = clock(); +} + +double Clock::end() { + end_t=clock(); + double runtime = (end_t-start_t)/double(CLOCKS_PER_SEC)*1000; + return runtime; +} + +double Clock::end(int iteration_num, std::string content) { + end_t=clock(); + double runtime=(end_t-start_t)/(double(CLOCKS_PER_SEC)*1000*iteration_num); + return runtime; +} + +void Clock::endWithLog( std::string content) { + double runtime=end(); + DLOG(ERROR)<<"Running time of "< Date: Wed, 27 Jan 2016 10:44:18 +0800 Subject: [PATCH 2/3] add clocks for forward and backward functions. --- Makefile.am | 4 +++- src/worker.cc | 15 ++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/Makefile.am b/Makefile.am index a432dc8f22..b80fba0d29 100644 --- a/Makefile.am +++ b/Makefile.am @@ -99,7 +99,8 @@ SINGA_SRCS := src/driver.cc \ src/utils/param.cc \ src/utils/updater.cc \ src/utils/blob.cc \ - src/utils/image_transform.cc + src/utils/image_transform.cc \ + src/utils/clock.cc SINGA_HDRS := include/singa.h \ include/singa/utils/math_blob.h \ @@ -117,6 +118,7 @@ SINGA_HDRS := include/singa.h \ include/utils/tinydir.h \ include/utils/tokenizer.h \ include/utils/image_transform.h \ + include/utils/clock.h \ include/server.h \ include/worker.h \ include/stub.h \ diff --git a/src/worker.cc b/src/worker.cc index 2afa8b06fe..91875f633a 100644 --- a/src/worker.cc +++ b/src/worker.cc @@ -30,13 +30,14 @@ #include "singa/utils/singleton.h" #include "singa/utils/context.h" #include "singa/utils/math_blob.h" +#include "singa/utils/clock.h" namespace singa { using std::string; Worker* Worker::Create(const AlgProto& conf) { - auto factory = Singleton>::Instance(); + auto factory = Singleton>::Instance(); Worker* worker = nullptr; if (conf.has_user_alg()) worker = factory->Create(conf.user_alg()); @@ -335,6 +336,8 @@ void BPWorker::TestOneBatch(int step, Phase phase, NeuralNet* net) { } void BPWorker::Forward(int step, Phase phase, NeuralNet* net) { + Clock c; + c.start(); map label; for (auto& layer : net->layers()) { if (layer->partition_id() == id_) { @@ -355,9 +358,12 @@ void BPWorker::Forward(int step, Phase phase, NeuralNet* net) { + std::to_string(step) +"-loc" + std::to_string(id_) + ".json"; WriteStringToTextFile(path, net->ToGraph(false).ToJson(label)); } + c.endWithLog("BPWorker::Forward"); } void BPWorker::Backward(int step, NeuralNet* net) { + Clock c; + c.start(); map label; auto& layers = net->layers(); for (auto it = layers.rbegin(); it != layers.rend(); it++) { @@ -375,10 +381,13 @@ void BPWorker::Backward(int step, NeuralNet* net) { + std::to_string(step) + "-loc" + std::to_string(id_) + ".json"; WriteStringToTextFile(path, net->ToGraph(false).Reverse().ToJson(label)); } + c.endWithLog("BPWorker::Backward"); } /***************************BPTTWorker*********************************/ void BPTTWorker::Forward(int step, Phase phase, NeuralNet* net) { + Clock c; + c.start(); map label; for (auto& layer : net->layers()) { if (layer->partition_id() == id_) { @@ -425,9 +434,12 @@ void BPTTWorker::Forward(int step, Phase phase, NeuralNet* net) { + std::to_string(step) +"-loc" + std::to_string(id_) + ".json"; WriteStringToTextFile(path, net->ToGraph(false).ToJson(label)); } +c.endWithLog("BPTTWorker::Forward"); } void BPTTWorker::Backward(int step, NeuralNet* net) { + Clock c; + c.start(); map label; auto& layers = net->layers(); for (auto it = layers.rbegin(); it != layers.rend(); it++) { @@ -449,6 +461,7 @@ void BPTTWorker::Backward(int step, NeuralNet* net) { + std::to_string(step) + "-loc" + std::to_string(id_) + ".json"; WriteStringToTextFile(path, net->ToGraph(false).Reverse().ToJson(label)); } +c.endWithLog("BPTTWorker::Backward"); } void BPTTWorker::Display(int flag, const std::string& prefix, NeuralNet* net) { std::unordered_map perf; From 2a8add2963988cf6644c81afebda5c62b5194a74 Mon Sep 17 00:00:00 2001 From: daijian Date: Tue, 15 Mar 2016 11:14:29 +0800 Subject: [PATCH 3/3] Performance profiling --- include/singa/utils/clock.h | 25 +++++++++----- src/utils/clock.cc | 65 +++++++++++++++++++++++++++---------- src/worker.cc | 14 ++++---- 3 files changed, 71 insertions(+), 33 deletions(-) diff --git a/include/singa/utils/clock.h b/include/singa/utils/clock.h index 785f48f9cf..52f5e3d2b5 100644 --- a/include/singa/utils/clock.h +++ b/include/singa/utils/clock.h @@ -19,8 +19,8 @@ * *************************************************************/ -#ifndef SINGA_CLOCK_H_ -#define SINGA_CLOCK_H_ +#ifndef _SINGA_UTILS_CLOCK_H_ +#define _SINGA_UTILS_CLOCK_H_ #include @@ -28,21 +28,28 @@ namespace singa { class Clock { public: - void start(); + Clock(); - double end(); - double end(int iteration_num, std::string content); - void endWithLog(std::string content); - void endWithLog(int iteration_num, std::string content); + void Start(); + + double End(); + double End(int iteration_num, std::string content); + void EndWithLog(std::string content); + void EndWithLog(int iteration_num, std::string content); + + double Elapse(); + double Elapse(int iteration_num, std::string content); + void ElapseWithLog(std::string content); + void ElapseWithLog(int iteration_num, std::string content); protected: int start_t = -1, end_t = -1; }; -} +} // namespace singa -#endif +#endif // SINGA_UTILS_CLOCK_H_ diff --git a/src/utils/clock.cc b/src/utils/clock.cc index b14f11fcb7..87bd40c837 100644 --- a/src/utils/clock.cc +++ b/src/utils/clock.cc @@ -31,31 +31,62 @@ namespace singa { -void Clock::start() { - start_t = clock(); +Clock::Clock() { + start_t = clock(); } -double Clock::end() { - end_t=clock(); - double runtime = (end_t-start_t)/double(CLOCKS_PER_SEC)*1000; - return runtime; + + +void Clock::Start() { + start_t = clock(); +} + +double Clock::End() { + end_t = clock(); + double runtime = (end_t-start_t)/static_cast(CLOCKS_PER_SEC)*1000; + return runtime; +} + +double Clock::End(int iteration_num, std::string content) { + end_t = clock(); + double runtime = (end_t-start_t); + runtime = runtime/(static_cast(CLOCKS_PER_SEC)*1000*iteration_num); + return runtime; } -double Clock::end(int iteration_num, std::string content) { - end_t=clock(); - double runtime=(end_t-start_t)/(double(CLOCKS_PER_SEC)*1000*iteration_num); - return runtime; +void Clock::EndWithLog(std::string content) { + double runtime = end(); + DLOG(ERROR) << "Running time of " << content<< " is " << runtime << " ms"; } -void Clock::endWithLog( std::string content) { - double runtime=end(); - DLOG(ERROR)<<"Running time of "<(CLOCKS_PER_SEC)*1000; + return runtime; } +double Clock::Elapse(int iteration_num, std::string content) { + end_t = clock(); + double runtime = (end_t-start_t); + runtime = runtime/(static_cast(CLOCKS_PER_SEC)*1000*iteration_num); + return runtime; } + +void Clock::ElapseWithLog(std::string content) { + double runtime = end(); + DLOG(ERROR)<< "Running time of " < label; for (auto& layer : net->layers()) { if (layer->partition_id() == id_) { @@ -358,12 +358,12 @@ void BPWorker::Forward(int step, Phase phase, NeuralNet* net) { + std::to_string(step) +"-loc" + std::to_string(id_) + ".json"; WriteStringToTextFile(path, net->ToGraph(false).ToJson(label)); } - c.endWithLog("BPWorker::Forward"); + c.endWithLog("BPWorker::Forward"); } void BPWorker::Backward(int step, NeuralNet* net) { Clock c; - c.start(); + c.start(); map label; auto& layers = net->layers(); for (auto it = layers.rbegin(); it != layers.rend(); it++) { @@ -387,7 +387,7 @@ void BPWorker::Backward(int step, NeuralNet* net) { /***************************BPTTWorker*********************************/ void BPTTWorker::Forward(int step, Phase phase, NeuralNet* net) { Clock c; - c.start(); + c.start(); map label; for (auto& layer : net->layers()) { if (layer->partition_id() == id_) { @@ -434,12 +434,12 @@ void BPTTWorker::Forward(int step, Phase phase, NeuralNet* net) { + std::to_string(step) +"-loc" + std::to_string(id_) + ".json"; WriteStringToTextFile(path, net->ToGraph(false).ToJson(label)); } -c.endWithLog("BPTTWorker::Forward"); + c.endWithLog("BPTTWorker::Forward"); } void BPTTWorker::Backward(int step, NeuralNet* net) { Clock c; - c.start(); + c.start(); map label; auto& layers = net->layers(); for (auto it = layers.rbegin(); it != layers.rend(); it++) { @@ -461,7 +461,7 @@ void BPTTWorker::Backward(int step, NeuralNet* net) { + std::to_string(step) + "-loc" + std::to_string(id_) + ".json"; WriteStringToTextFile(path, net->ToGraph(false).Reverse().ToJson(label)); } -c.endWithLog("BPTTWorker::Backward"); + c.endWithLog("BPTTWorker::Backward"); } void BPTTWorker::Display(int flag, const std::string& prefix, NeuralNet* net) { std::unordered_map perf;