layer_need_backward_ is now correct when propagate_down is specified

- skip_propagate_down is now called propagate_now
- the graph of the net is visited top-down to correctly update
layer_need_backward_
- tests updated

Author: mtamburrano

include/caffe/net.hpp

@@ -137,6 +137,9 @@ class Net {
   inline const vector<Dtype>& blob_loss_weights() const {
     return blob_loss_weights_;
   }
+  inline const vector<bool>& layer_need_backward() const {
+    return layer_need_backward_;
+  }
   /// @brief returns the parameters
   inline const vector<shared_ptr<Blob<Dtype> > >& params() const {
     return params_;
@@ -192,8 +195,7 @@ class Net {
   /// @brief Append a new bottom blob to the net.
   int AppendBottom(const NetParameter& param, const int layer_id,
                    const int bottom_id, set<string>* available_blobs,
-                   map<string, int>* blob_name_to_idx,
-                   bool skip_propagate = false);
+                   map<string, int>* blob_name_to_idx);
   /// @brief Append a new parameter blob to the net.
   void AppendParam(const NetParameter& param, const int layer_id,
                    const int param_id);

src/caffe/net.cpp

@@ -79,14 +79,12 @@ void Net<Dtype>::Init(const NetParameter& in_param) {
     }
     // Setup layer.
     const LayerParameter& layer_param = param.layer(layer_id);
-
-    if (layer_param.skip_propagate_down_size() > 0) {
-      CHECK_EQ(layer_param.skip_propagate_down_size(),
+    if (layer_param.propagate_down_size() > 0) {
+      CHECK_EQ(layer_param.propagate_down_size(),
           layer_param.bottom_size())
-      << "skip_propagate_down param must be specified"
+      << "propagate_down param must be specified"
         << "either 0 or bottom_size times ";
     }
-
     layers_.push_back(LayerRegistry<Dtype>::CreateLayer(layer_param));
     layer_names_.push_back(layer_param.name());
     LOG(INFO) << "Creating Layer " << layer_param.name();
@@ -95,13 +93,8 @@ void Net<Dtype>::Init(const NetParameter& in_param) {
     // Figure out this layer's input and output
     for (int bottom_id = 0; bottom_id < layer_param.bottom_size();
          ++bottom_id) {
-      bool skip_propagate_down = false;
-      // Check if the backpropagation on bottom_id should be skipped
-      if (layer_param.skip_propagate_down_size() > 0)
-        skip_propagate_down = layer_param.skip_propagate_down(bottom_id);
       const int blob_id = AppendBottom(param, layer_id, bottom_id,
-                                       &available_blobs, &blob_name_to_idx,
-                                       skip_propagate_down);
+                                       &available_blobs, &blob_name_to_idx);
       // If a blob needs backward, this layer should provide it.
       need_backward |= blob_need_backward_[blob_id];
     }
@@ -165,15 +158,33 @@ void Net<Dtype>::Init(const NetParameter& in_param) {
   // Go through the net backwards to determine which blobs contribute to the
   // loss.  We can skip backward computation for blobs that don't contribute
   // to the loss.
+  // Also checks if all bottom blobs don't need backward computation (possible
+  // because the skip_propagate_down param) and so we can skip bacward
+  // computation for the entire layer
   set<string> blobs_under_loss;
+  set<string> blobs_skip_backp;
   for (int layer_id = layers_.size() - 1; layer_id >= 0; --layer_id) {
     bool layer_contributes_loss = false;
+    bool layer_skip_propagate_down = true;
     for (int top_id = 0; top_id < top_vecs_[layer_id].size(); ++top_id) {
       const string& blob_name = blob_names_[top_id_vecs_[layer_id][top_id]];
       if (layers_[layer_id]->loss(top_id) ||
           (blobs_under_loss.find(blob_name) != blobs_under_loss.end())) {
         layer_contributes_loss = true;
+      }
+      if (blobs_skip_backp.find(blob_name) == blobs_skip_backp.end()) {
+        layer_skip_propagate_down = false;
+      }
+      if (layer_contributes_loss && !layer_skip_propagate_down)
         break;
+    }
+    // If this layer can skip backward computation, also all his bottom blobs
+    // don't need backpropagation
+    if (layer_need_backward_[layer_id] && layer_skip_propagate_down) {
+      layer_need_backward_[layer_id] = false;
+      for (int bottom_id = 0; bottom_id < bottom_vecs_[layer_id].size();
+               ++bottom_id) {
+        bottom_need_backward_[layer_id][bottom_id] = false;
       }
     }
     if (!layer_contributes_loss) { layer_need_backward_[layer_id] = false; }
@@ -192,6 +203,11 @@ void Net<Dtype>::Init(const NetParameter& in_param) {
       } else {
         bottom_need_backward_[layer_id][bottom_id] = false;
       }
+      if (!bottom_need_backward_[layer_id][bottom_id]) {
+        const string& blob_name =
+                   blob_names_[bottom_id_vecs_[layer_id][bottom_id]];
+        blobs_skip_backp.insert(blob_name);
+      }
     }
   }
   // Handle force_backward if needed.
@@ -383,7 +399,7 @@ void Net<Dtype>::AppendTop(const NetParameter& param, const int layer_id,
 template <typename Dtype>
 int Net<Dtype>::AppendBottom(const NetParameter& param, const int layer_id,
     const int bottom_id, set<string>* available_blobs,
-    map<string, int>* blob_name_to_idx, bool skip_propagate_down) {
+    map<string, int>* blob_name_to_idx) {
   const LayerParameter& layer_param = param.layer(layer_id);
   const string& blob_name = layer_param.bottom(bottom_id);
   if (available_blobs->find(blob_name) == available_blobs->end()) {
@@ -395,8 +411,12 @@ int Net<Dtype>::AppendBottom(const NetParameter& param, const int layer_id,
   bottom_vecs_[layer_id].push_back(blobs_[blob_id].get());
   bottom_id_vecs_[layer_id].push_back(blob_id);
   available_blobs->erase(blob_name);
+  bool propagate_down = true;
+  // Check if the backpropagation on bottom_id should be skipped
+  if (layer_param.propagate_down_size() > 0)
+    propagate_down = layer_param.propagate_down(bottom_id);
   const bool need_backward = blob_need_backward_[blob_id] &
-                          !skip_propagate_down;
+                          propagate_down;
   bottom_need_backward_[layer_id].push_back(need_backward);
   return blob_id;
 }

src/caffe/proto/caffe.proto

@@ -283,7 +283,7 @@ message LayerParameter {
 
   // Specifies on which bottoms the backpropagation should be skipped.
   // The size must be either 0 or equals to the number of bottoms.
-  repeated bool skip_propagate_down = 11;
+  repeated bool propagate_down = 11;
 
   // Rules controlling whether and when a layer is included in the network,
   // based on the current NetState.  You may specify a non-zero number of rules

src/caffe/test/test_net.cpp

@@ -55,8 +55,6 @@ class NetTest : public MultiDeviceTest<TypeParam> {
     }
   }
 
-
-
   virtual void InitTinyNet(const bool force_backward = false,
                            const bool accuracy_layer = false) {
     string proto =
@@ -701,9 +699,9 @@ class NetTest : public MultiDeviceTest<TypeParam> {
       "  bottom: 'innerproduct' "
       "  bottom: 'label_argmax' ";
     if (test_skip_true)
-      proto += "  skip_propagate_down: [false, true] ";
+      proto += "  propagate_down: [true, false] ";
     else
-      proto += "  skip_propagate_down: [false, false] ";
+      proto += "  propagate_down: [true, true] ";
     proto +=
       "  top: 'cross_entropy_loss' "
       "  type: 'SigmoidCrossEntropyLoss' "
@@ -2324,29 +2322,50 @@ TYPED_TEST(NetTest, TestReshape) {
 }
 
 TYPED_TEST(NetTest, TestSkipPropagateDown) {
-  // check bottom_need_backward if skip_propagat_down is false
+  // check bottom_need_backward if propagate_down is true
   this->InitSkipPropNet(false);
+  vector<bool> vec_layer_need_backward = this->net_->layer_need_backward();
   for (int layer_id = 0; layer_id < this->net_->layers().size(); ++layer_id) {
     if (this->net_->layer_names()[layer_id] == "loss") {
       // access to bottom_need_backward coresponding to label's blob
       bool need_back = this->net_->bottom_need_backward()[layer_id][1];
-      // if skip_propagate_down is false, the loss layer will try to
+      // if propagate_down is true, the loss layer will try to
       // backpropagate on labels
       CHECK_EQ(need_back, true)
         << "bottom_need_backward should be True";
     }
+    if (this->net_->layer_names()[layer_id] == "ip_fake_labels")
+      CHECK_EQ(vec_layer_need_backward[layer_id], true)
+        << "layer_need_backward for ip_fake_labels should be True";
+    if (this->net_->layer_names()[layer_id] == "argmax")
+      CHECK_EQ(vec_layer_need_backward[layer_id], true)
+        << "layer_need_backward for argmax should be True";
+    if (this->net_->layer_names()[layer_id] == "innerproduct")
+      CHECK_EQ(vec_layer_need_backward[layer_id], true)
+        << "layer_need_backward for innerproduct should be True";
   }
-  // check bottom_need_backward if skip_propagat_down is true
+  // check bottom_need_backward if propagat_down is false
   this->InitSkipPropNet(true);
+  vec_layer_need_backward.clear();
+  vec_layer_need_backward = this->net_->layer_need_backward();
   for (int layer_id = 0; layer_id < this->net_->layers().size(); ++layer_id) {
     if (this->net_->layer_names()[layer_id] == "loss") {
       // access to bottom_need_backward coresponding to label's blob
       bool need_back = this->net_->bottom_need_backward()[layer_id][1];
-      // if skip_propagate_down is true, the loss layer will not try to
+      // if propagate_down is false, the loss layer will not try to
       // backpropagate on labels
       CHECK_EQ(need_back, false)
         << "bottom_need_backward should be False";
     }
+    if (this->net_->layer_names()[layer_id] == "ip_fake_labels")
+      CHECK_EQ(vec_layer_need_backward[layer_id], false)
+        << "layer_need_backward for ip_fake_labels should be False";
+    if (this->net_->layer_names()[layer_id] == "argmax")
+      CHECK_EQ(vec_layer_need_backward[layer_id], false)
+        << "layer_need_backward for argmax should be False";
+    if (this->net_->layer_names()[layer_id] == "innerproduct")
+      CHECK_EQ(vec_layer_need_backward[layer_id], true)
+        << "layer_need_backward for innerproduct should be True";
   }
 }