小文件治理之hadoop集群监控：文件目录个数获取的三种方法

系列文章目录提示：这里可以添加系列文章的所有文章的目录，目录需要自己手动添加例如：第一章 Python 机器学习入门之pandas的使用提示：写完文章后，目录可以自动生成，如何生成可参考右边的帮助文档文章目录系列文章目录前言一、二、jmx获取二、使用hadoop fs -count 获取三、代码实现总结前言提示：这里可以添加本文要记录的大概内容：例如：随着人工智能的不断发展，机器学习这门技术也越来

星星之火

1046人浏览 · 2022-03-15 08:06:38

*星星之火* · 2022-03-15 08:06:38 发布

前言

总结一下获取文件目录数的方法

一、50070页面

在这里插入图片描述

二、jmx获取

jmx的服务端是servlet，通过httpClient调用就行


[root@hadoop101 software]# curl hadoop101:50070/jmx?qry=Hadoop:service=NameNode,name=NameNodeInfo
{
  "beans" : [ {
    "name" : "Hadoop:service=NameNode,name=NameNodeInfo",
    "modelerType" : "org.apache.hadoop.hdfs.server.namenode.FSNamesystem",
    "Total" : 3170276474880,
    "UpgradeFinalized" : true,
    "ClusterId" : "CID-f451766f-cd7d-4ade-994c-cc5359e5a60e",
    "Version" : "2.7.2, rb165c4fe8a74265c792ce23f546c64604acf0e41",
    "Used" : 91505946150,
    "Free" : 2914452942848,
    "Safemode" : "",
    "NonDfsUsedSpace" : 164317585882,
    "PercentUsed" : 2.8863711,
    "BlockPoolUsedSpace" : 91505946150,
    "PercentBlockPoolUsed" : 2.8863711,
    "PercentRemaining" : 91.930565,
    "CacheCapacity" : 0,
    "CacheUsed" : 0,
    "TotalBlocks" : 65507,
    "TotalFiles" : 66218,
    "NumberOfMissingBlocks" : 0,
    "NumberOfMissingBlocksWithReplicationFactorOne" : 0,
    "LiveNodes" : "{\"hadoop102:50010\":{\"infoAddr\":\"10.233.65.29:50075\",\"infoSecureAddr\":\"10.233.65.29:0\",\"xferaddr\":\"10.233.65.29:50010\",\"lastContact\":1,\"usedSpace\":26866655231,\"adminState\":\"In Service\",\"nonDfsUsedSpace\":54853758977,\"capacity\":1056758824960,\"numBlocks\":40849,\"version\":\"2.7.2\",\"used\":26866655231,\"remaining\":975038410752,\"blockScheduled\":0,\"blockPoolUsed\":26866655231,\"blockPoolUsedPercent\":2.5423641,\"volfails\":0},\"hadoop101:50010\":{\"infoAddr\":\"10.233.65.15:50075\",\"infoSecureAddr\":\"10.233.65.15:0\",\"xferaddr\":\"10.233.65.15:50010\",\"lastContact\":2,\"usedSpace\":23621656822,\"adminState\":\"In Service\",\"nonDfsUsedSpace\":54771056394,\"capacity\":1056758824960,\"numBlocks\":53006,\"version\":\"2.7.2\",\"used\":23621656822,\"remaining\":978366111744,\"blockScheduled\":0,\"blockPoolUsed\":23621656822,\"blockPoolUsedPercent\":2.2352931,\"volfails\":0},\"hadoop103:50010\":{\"infoAddr\":\"10.233.65.40:50075\",\"infoSecureAddr\":\"10.233.65.40:0\",\"xferaddr\":\"10.233.65.40:50010\",\"lastContact\":2,\"usedSpace\":41017634097,\"adminState\":\"In Service\",\"nonDfsUsedSpace\":54692770511,\"capacity\":1056758824960,\"numBlocks\":37161,\"version\":\"2.7.2\",\"used\":41017634097,\"remaining\":961048420352,\"blockScheduled\":0,\"blockPoolUsed\":41017634097,\"blockPoolUsedPercent\":3.8814566,\"volfails\":0}}",
    "DeadNodes" : "{}",
    "DecomNodes" : "{}",
    "BlockPoolId" : "BP-1805414249-10.233.65.15-1641883882370",
    "NameDirStatuses" : "{\"active\":{\"/data/hadoop/hdfs/name\":\"IMAGE_AND_EDITS\"},\"failed\":{}}",
    "NodeUsage" : "{\"nodeUsage\":{\"min\":\"2.24%\",\"median\":\"2.54%\",\"max\":\"3.88%\",\"stdDev\":\"0.71%\"}}",
    "NameJournalStatus" : "[{\"manager\":\"QJM to [10.233.65.15:8485, 10.233.65.29:8485, 10.233.65.40:8485]\",\"stream\":\"open for read\",\"disabled\":\"false\",\"required\":\"true\"}]",
    "JournalTransactionInfo" : "{\"MostRecentCheckpointTxId\":\"4931140\",\"LastAppliedOrWrittenTxId\":\"4936276\"}",
    "NNStarted" : "Wed Jan 26 16:54:26 CST 2022",
    "CompileInfo" : "2016-01-26T00:08Z by jenkins from (detached from b165c4f)",
    "CorruptFiles" : "[]",
    "DistinctVersionCount" : 1,
    "DistinctVersions" : [ {
      "key" : "2.7.2",
      "value" : 3
    } ],
    "SoftwareVersion" : "2.7.2",
    "RollingUpgradeStatus" : null,
    "Threads" : 37
  } ]
}[

三、使用hadoop fs -count 获取

[root@hadoop101 software]#  hadoop fs -count   /
         714        65506        44795164671 /

总结

介绍了文件目录树可以通过web,jmx,hadoop fs -count获取

天启AI社区

GitCode 天启AI是一款由 GitCode 团队打造的智能助手，基于先进的LLM（大语言模型）与多智能体 Agent 技术构建，致力于为用户提供高效、智能、多模态的创作与开发支持。它不仅支持自然语言对话，还具备处理文件、生成 PPT、撰写分析报告、开发 Web 应用等多项能力，真正做到“一句话，让 Al帮你完成复杂任务”。

更多推荐