hadoop自定义权限

2018-12-07 16:26 
版权声明:本文为博主九师兄(QQ群:spark源代码 198279782 欢迎来探讨技术)原创文章,未经博主允许不得转载。 https://blog.csdn.net/qq_21383435/article/details/81087908

参考:https://blog.csdn.net/houzhizhen/article/details/78363332

#1.测试类
想运行hadoop的测试类,我们必须先编译hadoop

mac下编译hadoop-2.7.4

然后查看测试类

org.apache.hadoop.hdfs.server.namenode.TestINodeAttributeProvider

然后直接运行testDelegationToProvider这个测试类方法,就可以运行,并且在

@Override
      public void checkPermission(String fsOwner, String supergroup,
          UserGroupInformation ugi, INodeAttributes[] inodeAttrs,
          INode[] inodes, byte[][] pathByNameArr, int snapshotId, String path,
          int ancestorIndex, boolean doCheckOwner, FsAction ancestorAccess,
          FsAction parentAccess, FsAction access, FsAction subAccess,
          boolean ignoreEmptyDir) throws AccessControlException {
          
          System.out.println("=========="+fsOwner);
        System.out.println("=========="+supergroup);
        System.out.println("=========="+ugi);
        System.out.println("=========="+inodeAttrs);
        System.out.println("=========="+inodes);
        System.out.println("=========="+pathByNameArr);
        System.out.println("=========="+snapshotId);
        System.out.println("=========="+path);
        System.out.println("=========="+ancestorIndex);
        System.out.println("=========="+doCheckOwner);
        System.out.println("=========="+ancestorAccess);
        System.out.println("=========="+parentAccess);
        System.out.println("=========="+access);
        System.out.println("=========="+subAccess);
        System.out.println("=========="+ignoreEmptyDir);
}

方法中打印出我们需要的内容

/**
         fsOwner = lcc
         supergroup = supergroup
         ugi = u1 (auth:SIMPLE)
         inodeAttrs = [Lorg.apache.hadoop.hdfs.server.namenode.INodeAttributes;@6a93a149
         inodes = [Lorg.apache.hadoop.hdfs.server.namenode.INode;@31d3da26
         pathByNameArr = [[B@35308ff8
         snapshotId = 2147483646
         path = /tmp/foo
         ancestorIndex = 1
         doCheckOwner = false
         ancestorAccess = null
         parentAccess = null
         access = READ_EXECUTE
         subAccess = null
         ignoreEmptyDir = false
         */

但是注意看方法

 @Before
  public void setUp() throws IOException {
    CALLED.clear();
    Configuration conf = new HdfsConfiguration();
    String name = MegrezHdfsAuthorizer.class.getName();
    conf.set(DFSConfigKeys.DFS_NAMENODE_INODE_ATTRIBUTES_PROVIDER_KEY,
        MyAuthorizationProvider.class.getName());
//            name);
    conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true);
    EditLogFileOutputStream.setShouldSkipFsyncForTesting(true);
    miniDFS = new MiniDFSCluster.Builder(conf).build();
  }

这个方法设置了两个重要的属性

 public static final String  DFS_NAMENODE_INODE_ATTRIBUTES_PROVIDER_KEY = "dfs.namenode.inode.attributes.provider.class";
   public static final String  DFS_NAMENODE_ACLS_ENABLED_KEY = "dfs.namenode.acls.enabled";

2.不修改源代码,实现自定义权限

新建一个maven项目,在/Users/lcc/IdeaProjects/hadoop-hdfs/src/main/java/目录下新建一个包package org.apache.hadoop.hdfs.server.namenode;(这个包位置很重要,一定要是和org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider这个类一样的目录),然后新建类MyHdfsAuthorizer.java
书写代码

package org.apache.hadoop.hdfs.server.namenode;


import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.security.UserGroupInformation;

import java.security.AccessControlException;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;


/**
 * HDFS 鉴权具体的实现
 *
 * @author hulb
 * @date 2018/7/11 下午3:17
 * <p>
 * cp hdfs-auth/target/hdfs-auth-spark2.0.jar  /Users/hulb/opt/third/hadoop/share/hadoop/hdfs/lib/
 * <p>
 * 实现思路:
 * 还需要加一个权限列表的缓存。
 */
public class MyHdfsAuthorizer extends INodeAttributeProvider {

    private static final Log LOG = LogFactory.getLog(MegrezHdfsAuthorizer.class);


    @Override
    public void start() {
        System.out.println("MegrezHdfsAuthorizer 启动");
    }

    @Override
    public void stop() {
        System.out.println("MegrezHdfsAuthorizer 停止");
    }

    /**
     * 必须重写的方法
     * @param deafultEnforcer
     * @return
     */
    @Override
    public AccessControlEnforcer getExternalAccessControlEnforcer(
            AccessControlEnforcer deafultEnforcer) {
        return new MegrezAccessControlEnforcer();
    }


    /**
     * cc@lcc ~$ hdfs dfs -ls /user/hive /user/lcc
     *
     * @param pathElements 这里的fullPath是我们操作Hdfs时候的全路径
     *                 比如:        hdfs dfs -ls /
     *                 那么全路径是
     *                 /
     *                 /hadoop
     *                 /ranger
     *                 比如:        hdfs dfs -ls /
     *                 那么全路径是
     *                 /hadoop/
     *                 /hadoop/tempdata
     * @param inode    路径:/hadoop/tempdata
     *                 getGroupName            : supergroup
     *                 getUserName             : lcc
     *                 getAccessTime           : 0
     *                 getAclFeature           : null
     *                 getFsPermission         : rwxr-xr-x
     *                 getFsPermissionShort    : 493
     *                 getLocalNameBytes       : [B@50af74e1
     *                 getModificationTime     : 1530179365668
     *                 getXAttrFeature         : null
     *                 isDirectory             : true
     *                 toString                : tempdata
     * @return
     */

    @Override
    public INodeAttributes getAttributes(String[] pathElements,
                                          INodeAttributes inode) {
//        for(int i=0;i<pathElements.length;i++){
//            System.out.println(pathElements[i].toString());
//        }
        return inode;
    }



    public  class MegrezAccessControlEnforcer implements AccessControlEnforcer {


        private ConcurrentMap<String, UserAction> allPass =
                new ConcurrentHashMap<String, UserAction>();

		/**
	     * 必须重写的方法
	     * 改方法会被系统自动调用
	     * /
        @Override
        public void checkPermission(String fsOwner, String supergroup,
                                    UserGroupInformation ugi, INodeAttributes[] inodeAttrs,
                                    INode[] inodes, byte[][] pathByNameArr, int snapshotId, String path,
                                    int ancestorIndex, boolean doCheckOwner, FsAction ancestorAccess,
                                    FsAction parentAccess, FsAction access, FsAction subAccess,
                                    boolean ignoreEmptyDir) throws AccessControlException {

				权限鉴定业务代码

            }
        }
    }


}

然后修改$HADOOP_HOME/etc/hadoop/hdfs-site.xml

<configuration>

        <property>
                 <name>dfs.replication</name>
                 <value>1</value>
        </property>
        <property>
                <name>dfs.permissions</name>
                <value>false</value>
        </property>
        <property>
                <name>dfs.permissions</name>
                <value>true</value>
        </property>

        <property>
                <name>dfs.permissions.umask-mode</name>
                <value>077</value>
         </property>
        <property>
                <name>dfs.permissions.enabled</name>
                <value>true</value>
         </property>
        <property>
                <name>dfs.namenode.acls.enabled</name>
                <value>true</value>
         </property>
        <property>
                <name>dfs.namenode.inode.attributes.provider.class</name>
                <value>org.apache.hadoop.hdfs.server.namenode.MyHdfsAuthorizer</value>
        </property>

</configuration>

然后打包这个项目,然后拷贝到本地hdaoop集群的share目录下

lcc@lcc ~$ cp ~/IdeaProjects/spark-authorizer/hadoop_hdfs/target/hdfs-auth-spark2.0.jar /Users/lcc/soft/hadoop/hadoop/share/hadoop/hdfs/lib/

然后重启集群

lcc@lcc hadoop$ sbin/start-all.sh
This script is Deprecated. Instead use start-dfs.sh and start-yarn.sh
18/07/16 15:44:59 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
Starting namenodes on [lcc]
lcc: starting namenode, logging to /Users/lcc/soft/hadoop/hadoop/logs/hadoop-lcc-namenode-lcc.out
localhost: starting datanode, logging to /Users/lcc/soft/hadoop/hadoop/logs/hadoop-lcc-datanode-lcc.out
Starting secondary namenodes [0.0.0.0]
0.0.0.0: starting secondarynamenode, logging to /Users/lcc/soft/hadoop/hadoop/logs/hadoop-lcc-secondarynamenode-lcc.out
18/07/16 15:45:17 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
starting yarn daemons
starting resourcemanager, logging to /Users/lcc/soft/hadoop/hadoop/logs/yarn-lcc-resourcemanager-lcc.out
localhost: starting nodemanager, logging to /Users/lcc/soft/hadoop/hadoop/logs/yarn-lcc-nodemanager-lcc.out
lcc@lcc hadoop$

然后测试

lcc@lcc ~$ hdfs dfs -mkdir  /lcc6
18/07/16 14:27:36 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable

然后查看日志

lcc@lcc ~$ tail -100 /Users/lcc/soft/hadoop/hadoop/logs/hadoop-lcc-namenode-lcc.out
MegrezHdfsAuthorizer 启动
调用了本地重写的getAttributes方法
调用了本地重写的getAttributes方法
调用了本地重写的getAttributes方法
调用了本地重写的getAttributes方法
ax memory size         (kbytes, -m) unlimited
open files                      (-n) 256
pipe size            (512 bytes, -p) 1
stack size              (kbytes, -s) 8192
cpu time               (seconds, -t) unlimited
max user processes              (-u) 709
virtual memory          (kbytes, -v) unlimited

然后他妈的尴尬了,这里竟然每调用到内部类checkPermission方法

@Override
        public void checkPermission(String fsOwner, String supergroup,
                                    UserGroupInformation ugi, INodeAttributes[] inodeAttrs,
                                    INode[] inodes, byte[][] pathByNameArr, int snapshotId, String path,
                                    int ancestorIndex, boolean doCheckOwner, FsAction ancestorAccess,
                                    FsAction parentAccess, FsAction access, FsAction subAccess,
                                    boolean ignoreEmptyDir) throws AccessControlException {

这个问题已经解决,原来是使用命令不会调用,实用程序调用就可以了
比如

hdfs dfs -mkdir /lcc12

用程序调用就可以了


    <dependencies>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>3.8.1</version>
            <scope>test</scope>
        </dependency>

        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
            <version>2.7.4</version>
        </dependency>

        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>2.7.4</version>
        </dependency>

        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>2.7.4</version>
        </dependency>

    </dependencies>
package com.lcc.hadoop.test;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

import java.net.URI;

public class MyMkdir {
    public static void main(String[] args)throws Exception{
        FileSystem fs = FileSystem.get(new URI("hdfs://lcc:9000"),new Configuration(),"root");
        //测试创建一个文件夹,在HDFS上创建一个leitao文件夹,原根目录下使没有这个文件的
        boolean flag = fs.mkdirs(new Path("/leitao3"));
        System.out.println(flag);
    }
}

这是一个大坑,具体为撒不一样,可能是命令和FileSystem内部实现是不一样的

发表评论

您必须 登录 才能发表留言!