TEZ-4715: Migrate Tez PB Protocols from ProtobufRPCEngine to ProtobufRPCEngine2

Author: Aggarwal-Raghav

.codespellrc

@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+[codespell]
+ignore-words-list = thirdparty

pom.xml

@@ -77,6 +77,7 @@
     <frontend-maven-plugin.version>1.15.0</frontend-maven-plugin.version>
     <guava.version>32.0.1-jre</guava.version>
     <hadoop.version>3.4.2</hadoop.version>
+    <hadoop-thirdparty.version>1.4.0</hadoop-thirdparty.version>
     <jersey.version>1.19.4</jersey.version>
     <jettison.version>1.5.4</jettison.version>
     <jsr305.version>3.0.0</jsr305.version>
@@ -101,6 +102,7 @@
     <test.log.dir>${project.build.directory}/logs</test.log.dir>
     <wro4j-maven-plugin.version>1.7.9</wro4j-maven-plugin.version>
     <maven.javadoc.skip>true</maven.javadoc.skip> <!-- enabled only in relevant modules separately -->
+    <maven-replacer-plugin.version>1.5.3</maven-replacer-plugin.version>
   </properties>
   <scm>
     <connection>${scm.url}</connection>
@@ -773,6 +775,11 @@
         <artifactId>protobuf-java</artifactId>
         <version>${protobuf.version}</version>
       </dependency>
+      <dependency>
+        <groupId>org.apache.hadoop.thirdparty</groupId>
+        <artifactId>hadoop-shaded-protobuf_3_25</artifactId>
+        <version>${hadoop-thirdparty.version}</version>
+      </dependency>
       <dependency>
         <groupId>com.google.guava</groupId>
         <artifactId>guava</artifactId>
@@ -1100,6 +1107,51 @@
             </dependency>
           </dependencies>
         </plugin>
+        <plugin>
+          <groupId>com.google.code.maven-replacer-plugin</groupId>
+          <artifactId>replacer</artifactId>
+          <version>${maven-replacer-plugin.version}</version>
+          <executions>
+            <execution>
+              <id>replace-generated-sources</id>
+              <phase>generate-sources</phase>
+              <goals>
+                <goal>replace</goal>
+              </goals>
+              <configuration>
+                <basedir>${project.basedir}</basedir>
+                <includes>
+                  <include>target/generated-sources/**/*.java</include>
+                </includes>
+                <replacements>
+                  <replacement>
+                    <token>([^\.])com.google.protobuf</token>
+                    <value>$1org.apache.hadoop.thirdparty.protobuf</value>
+                  </replacement>
+                </replacements>
+              </configuration>
+            </execution>
+            <execution>
+              <id>replace-generated-test-sources</id>
+              <phase>generate-sources</phase>
+              <goals>
+                <goal>replace</goal>
+              </goals>
+              <configuration>
+                <basedir>${project.basedir}</basedir>
+                <includes>
+                  <include>target/generated-test-sources/**/*.java</include>
+                </includes>
+                <replacements>
+                  <replacement>
+                    <token>([^\.])com.google.protobuf</token>
+                    <value>$1org.apache.hadoop.thirdparty.protobuf</value>
+                  </replacement>
+                </replacements>
+              </configuration>
+            </execution>
+          </executions>
+        </plugin>
       </plugins>
     </pluginManagement>
 

tez-api/pom.xml

@@ -82,8 +82,8 @@
      <artifactId>commons-collections4</artifactId>
     </dependency>
     <dependency>
-     <groupId>com.google.protobuf</groupId>
-     <artifactId>protobuf-java</artifactId>
+     <groupId>org.apache.hadoop.thirdparty</groupId>
+     <artifactId>hadoop-shaded-protobuf_3_25</artifactId>
     </dependency>
     <dependency>
      <groupId>junit</groupId>
@@ -183,6 +183,10 @@
           </execution>
         </executions>
       </plugin>
+      <plugin>
+        <groupId>com.google.code.maven-replacer-plugin</groupId>
+        <artifactId>replacer</artifactId>
+      </plugin>
     </plugins>
   </build>
 

tez-api/src/main/java/org/apache/tez/client/FrameworkClient.java

@@ -23,6 +23,7 @@
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.thirdparty.protobuf.ServiceException;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
@@ -49,8 +50,6 @@
 import org.apache.tez.frameworkplugins.FrameworkUtils;
 import org.apache.tez.frameworkplugins.yarn.YarnClientFrameworkService;
 
-import com.google.protobuf.ServiceException;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 

tez-api/src/main/java/org/apache/tez/client/TezClient.java

@@ -44,6 +44,7 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.thirdparty.protobuf.ServiceException;
 import org.apache.hadoop.util.Time;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
@@ -80,7 +81,6 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.Maps;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import com.google.protobuf.ServiceException;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java

@@ -56,7 +56,7 @@
 import org.apache.hadoop.fs.permission.FsAction;
 import org.apache.hadoop.io.DataOutputBuffer;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.ProtobufRpcEngine2;
 import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.Credentials;
@@ -982,7 +982,7 @@ public static DAGClientAMProtocolBlockingPB getAMProxy(final Configuration conf,
       proxy = userUgi.doAs(new PrivilegedExceptionAction<DAGClientAMProtocolBlockingPB>() {
         @Override
         public DAGClientAMProtocolBlockingPB run() throws IOException {
-          RPC.setProtocolEngine(conf, DAGClientAMProtocolBlockingPB.class, ProtobufRpcEngine.class);
+          RPC.setProtocolEngine(conf, DAGClientAMProtocolBlockingPB.class, ProtobufRpcEngine2.class);
           return (DAGClientAMProtocolBlockingPB) RPC.getProxy(DAGClientAMProtocolBlockingPB.class,
               0, serviceAddr, conf);
         }

tez-api/src/main/java/org/apache/tez/common/RPCUtil.java

@@ -23,13 +23,12 @@
 import java.lang.reflect.InvocationTargetException;
 
 import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.thirdparty.protobuf.ServiceException;
 import org.apache.tez.dag.api.DAGNotRunningException;
 import org.apache.tez.dag.api.NoCurrentDAGException;
 import org.apache.tez.dag.api.SessionNotRunning;
 import org.apache.tez.dag.api.TezException;
 
-import com.google.protobuf.ServiceException;
-
 public final class RPCUtil {
 
   private RPCUtil() {}

tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java

@@ -44,15 +44,14 @@
 import org.apache.hadoop.io.DataOutputBuffer;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.thirdparty.protobuf.ByteString;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.tez.client.TezClient;
 import org.apache.tez.common.security.JobTokenIdentifier;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.TezConstants;
 import org.apache.tez.dag.api.TezUncheckedException;
 
-import com.google.protobuf.ByteString;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 

tez-api/src/main/java/org/apache/tez/common/TezUtils.java

@@ -28,15 +28,14 @@
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.thirdparty.protobuf.ByteString;
+import org.apache.hadoop.thirdparty.protobuf.CodedInputStream;
 import org.apache.tez.client.TezClientUtils;
 import org.apache.tez.dag.api.TezUncheckedException;
 import org.apache.tez.dag.api.UserPayload;
 import org.apache.tez.dag.api.records.DAGProtos;
 import org.apache.tez.runtime.api.TaskContext;
 
-import com.google.protobuf.ByteString;
-import com.google.protobuf.CodedInputStream;
-
 import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
 import org.slf4j.Logger;

tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java

@@ -38,6 +38,8 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.DataInputByteBuffer;
 import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.thirdparty.protobuf.ByteString;
+import org.apache.hadoop.thirdparty.protobuf.ByteString.Output;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.LocalResourceType;
 import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
@@ -86,8 +88,6 @@
 import org.apache.tez.serviceplugins.api.ServicePluginsDescriptor;
 
 import com.google.common.collect.Lists;
-import com.google.protobuf.ByteString;
-import com.google.protobuf.ByteString.Output;
 
 @Private
 public final class DagTypeConverters {