#ifndef CAFFE2_OPERATORS_SINUSOID_POSITION_ENCODING_OP_H_ #define CAFFE2_OPERATORS_SINUSOID_POSITION_ENCODING_OP_H_ #ifdef _MSC_VER #define _USE_MATH_DEFINES #endif // _MSC_VER #include #include "caffe2/core/operator.h" #include "Eigen/Core" #include "caffe2/utils/eigen_utils.h" namespace caffe2 { template class SinusoidPositionEncodingOp : public Operator { public: template explicit SinusoidPositionEncodingOp(Args&&... args) : Operator(std::forward(args)...), embedding_size_( this->template GetSingleArgument("embedding_size", 100)), alpha_(this->template GetSingleArgument("alpha", 10000)), amplitude_(this->template GetSingleArgument("amplitude", 1)) {} USE_OPERATOR_CONTEXT_FUNCTIONS; bool RunOnDevice() override { return DispatchHelper>::call( this, this->template Input(0, CPU)); } template bool DoRunWithType() { auto& positions = Input(0); CAFFE_ENFORCE_EQ(positions.dim(), 2, "POSITIONS should be a 2-D tensor"); auto shape = positions.sizes().vec(); shape.push_back(embedding_size_); auto* output = Output(0, shape, at::dtype()); int M = shape[0]; int K = shape[1]; const Index* idxs = positions.template data(); float* out = output->template mutable_data(); float log_alpha = std::log(alpha_); float max_alpha_pow = ((float)embedding_size_ - 1.0f) / (float)embedding_size_; for (int i = 0; i < M; ++i) { float pos = (float)idxs[i * K]; // Compute the embedding for position i, example 0 first float* row = &out[i * K * embedding_size_]; Eigen::Map row_map(row, embedding_size_, 1); auto row_array = row_map.array(); float log_pos = std::log(pos); row_array.setLinSpaced( embedding_size_, log_pos, log_pos - log_alpha * max_alpha_pow); row_array = row_array.exp().eval(); // row_array[k] == pos / alpha^(k / embedding_size) // Phase shift so that alternating elements are cosines for (int k = 1; k < embedding_size_; k += 2) { row[k] += (float)M_PI_2; } row_array = amplitude_ * row_array.sin().eval(); // Copy the embedding to position i in the other examples for (int j = 1; j < K; ++j) { int base = i * K * embedding_size_; std::copy( &out[base], &out[base + embedding_size_], &out[base + j * embedding_size_]); } } return true; } protected: int embedding_size_; float alpha_; float amplitude_; }; } // namespace caffe2 #endif // CAFFE2_OPERATORS_SINUSOID_POSITION_ENCODING_OP_H_