Week8-1
No files matched your search
|
After Width: | Height: | Size: 30 KiB |
@@ -0,0 +1,731 @@
|
|||||||
|
<!doctype html>
|
||||||
|
<html lang="zh-CN">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<meta name="color-scheme" content="light">
|
||||||
|
<meta name="description" content="第8周计量经济学研究报告:回归自由度与财务、治理数据合并。">
|
||||||
|
<title>第8周研究报告:回归自由度与财务、治理数据合并</title>
|
||||||
|
<style>
|
||||||
|
:root{--ink:#183344;--text:#293e4b;--muted:#657986;--teal:#147a78;--line:#dbe5e9;--paper:#fff;--bg:#f0f4f6;--serif:"Cambria Math","STIX Two Math","Times New Roman","Noto Serif SC",serif}
|
||||||
|
*{box-sizing:border-box}html{scroll-behavior:smooth;scroll-padding-top:28px}body{margin:0;background:var(--bg);color:var(--text);font-family:"Segoe UI","Microsoft YaHei","PingFang SC",sans-serif;font-size:16px;line-height:1.85}a{color:#126e76;text-decoration-thickness:1px;text-underline-offset:3px}a:hover{color:#0c4e58}button{font:inherit}a:focus-visible,button:focus-visible,[tabindex]:focus-visible{outline:3px solid #e5a543;outline-offset:4px}.skip-link{position:absolute;left:20px;top:-80px;padding:10px;background:white;z-index:10}.skip-link:focus{top:14px}.masthead{background:#153544;border-top:5px solid #2eaaa0;color:#fff}.masthead-inner{max-width:1260px;margin:auto;padding:42px 42px 38px}.eyebrow{font-size:12px;letter-spacing:.19em;color:#a9d8d7;font-weight:600;margin-bottom:14px}.masthead h1{font-size:clamp(26px,3.2vw,39px);font-weight:650;line-height:1.45;letter-spacing:.01em;max-width:1000px;margin:0}.masthead-bottom{display:flex;align-items:center;justify-content:space-between;gap:20px;margin-top:22px}.masthead-caption{color:#c5d6df;font-size:14px}.header-actions{display:flex;gap:10px;flex-wrap:wrap}.header-actions a,.header-actions button{border:1px solid #6b8c99;background:transparent;color:#edf6f8;border-radius:5px;padding:7px 14px;font-size:13px;text-decoration:none;cursor:pointer;line-height:1.5}.header-actions a:hover,.header-actions button:hover{background:#264d5d}.shell{display:grid;grid-template-columns:230px minmax(0,1fr);gap:34px;max-width:1260px;margin:34px auto 60px;padding:0 34px}.sidebar{min-width:0}.sidebar-inner{position:sticky;top:26px;border-top:2px solid #b1ccd0;padding-top:16px;max-height:calc(100vh - 52px);overflow:auto}.toc-title{font-size:12px;font-weight:700;letter-spacing:.12em;color:#59717d;padding:0 10px 10px}.toc{list-style:none;margin:0;padding:0}.toc li{margin:2px 0}.toc a{display:block;font-size:13px;line-height:1.65;text-decoration:none;padding:7px 10px;color:#4a6674;border-radius:4px;border-left:2px solid transparent}.toc a:hover{background:#e3edef;color:var(--teal);border-left-color:var(--teal)}.toc .level-3 a{padding-left:23px;font-size:12px;color:#6e838e}.sidebar-note{padding:16px 10px 0;margin-top:16px;border-top:1px solid var(--line);color:#738690;font-size:12px}.report{min-width:0;background:var(--paper);padding:36px 42px 42px;border:1px solid #e0e8ec;border-radius:8px;box-shadow:0 4px 18px #17374704}.report>p:first-child{margin-top:0}.report p{margin:15px 0}.report h2{font-size:25px;font-weight:650;line-height:1.5;color:var(--ink);margin:46px 0 18px;padding-top:8px;border-top:1px solid var(--line)}.report h2:first-child{margin-top:0}.report h3{font-size:19px;font-weight:650;line-height:1.65;color:#225466;margin:28px 0 12px}.report h4{font-size:16px;color:var(--ink);margin:22px 0 9px}.report strong{color:#173c4d;font-weight:650}.report ul,.report ol{padding-left:1.55em;margin:14px 0}.report li{padding-left:4px;margin:7px 0}.report li>p{margin:5px 0}.report blockquote{margin:22px 0;padding:13px 19px;border-left:4px solid #2e918e;background:#eef7f6;color:#355c64;border-radius:0 5px 5px 0}.report blockquote p{margin:5px 0}.report hr{border:0;border-top:1px solid var(--line);margin:32px 0}.report code{font-family:"Cascadia Code",Consolas,"Microsoft YaHei",monospace;font-size:.88em;background:#eff4f6;border-radius:3px;padding:2px 5px;overflow-wrap:anywhere}.code-block{margin:20px 0;border:1px solid #dbe5eb;border-radius:6px;overflow:hidden;background:#f5f8fa}.code-label{padding:6px 15px;background:#eaf0f4;border-bottom:1px solid #dbe5eb;font-size:11px;color:#607680;font-weight:650;text-transform:uppercase;letter-spacing:.06em}.report pre{margin:0;padding:16px 18px;line-height:1.65;overflow-x:auto;font-size:13px}.report pre code{padding:0;border:0;border-radius:0;background:transparent;font-size:inherit;overflow-wrap:normal;white-space:pre}.equation{padding:20px 24px;margin:22px 0;background:#fff;border:1px solid #cddfe2;border-left:3px solid #4e9594;border-radius:5px;font-family:var(--serif);font-size:19px;line-height:1.95;text-align:center;overflow-x:auto}.equation>div{min-width:max-content;white-space:pre}.table-scroll{overflow:auto;margin:20px 0 24px;border:1px solid #d9e4e8;border-radius:6px}.table-scroll table{border-collapse:collapse;min-width:100%;font-size:13px;line-height:1.65}.table-scroll th,.table-scroll td{padding:10px 13px;vertical-align:top;border-bottom:1px solid #e4ebee;text-align:left}.table-scroll th{font-weight:650;color:#294c5b;background:#eaf2f4;white-space:nowrap}.table-scroll td{min-width:70px}.table-scroll td code{font-size:12px}.table-scroll tbody tr:nth-child(even){background:#f8fafb}.table-scroll tbody tr:last-child td{border-bottom:0}.table-scroll tbody tr:hover{background:#edf6f5}.report img{display:block;max-width:100%;height:auto;margin:23px auto;border:1px solid #e0e8ec;border-radius:4px}.report a{overflow-wrap:anywhere}.report .footnotes{font-size:13px}.footer{max-width:1260px;margin:0 auto;padding:0 34px 28px;text-align:right;colLine truncated
|
||||||
|
.equation>div{min-width:0;white-space:pre-wrap;overflow-wrap:anywhere}
|
||||||
|
@media(min-width:1450px){.shell{gap:40px}.report{padding:40px 50px 46px}}
|
||||||
|
@media(max-width:950px){.shell{grid-template-columns:185px minmax(0,1fr);gap:20px;padding:0 22px}.report{padding:28px}.masthead-inner{padding:32px 28px}.toc a{font-size:12px}.toc .level-3{display:none}}
|
||||||
|
@media(max-width:720px){body{font-size:15px}.masthead-inner{padding:26px 21px}.masthead-bottom{align-items:flex-start;flex-direction:column;gap:13px}.masthead-caption{font-size:12px}.shell{display:block;margin:20px auto 30px;padding:0 14px}.sidebar{margin:0 2px 20px}.sidebar-inner{position:static;max-height:none;padding-top:12px}.toc-title{padding:0 3px 7px}.toc{display:flex;flex-wrap:wrap;gap:4px}.toc li{margin:0}.toc a{font-size:12px;background:#e5edef;padding:6px 10px}.toc .level-3,.sidebar-note{display:none}.report{padding:24px 19px;border-radius:6px}.report h2{font-size:22px;margin-top:35px}.report h3{font-size:18px}.equation{font-size:17px;padding:14px 16px}.table-scroll th,.table-scroll td{padding:9px 11px}.footer{padding:0 20px 22px;text-align:left}}
|
||||||
|
@media print{@page{size:A4;margin:17mm 16mm}html{scroll-behavior:auto}body{background:white;color:#111;font-size:10.5pt;line-height:1.65}.masthead{background:white!important;color:#122b38;border-top:3px solid #468d88}.masthead-inner{padding:14px 0 19px}.eyebrow{font-size:8pt;color:#486b72;margin-bottom:8px}.masthead h1{font-size:23pt}.masthead-bottom{margin-top:12px}.masthead-caption{font-size:9pt;color:#526b76}.header-actions,.sidebar,.skip-link,.footer{display:none!important}.shell{display:block;margin:0;padding:0;max-width:none}.report{padding:0;border:0;box-shadow:none;border-radius:0}.report h2{font-size:16pt;break-after:avoid;margin-top:24px}.report h3{font-size:12pt;break-after:avoid}.report p,.report li{orphans:3;widows:3}.report a{color:inherit;text-decoration:none}.report pre{white-space:pre-wrap;font-size:8pt;overflow:visible}.report pre code{white-space:pre-wrap;overflow-wrap:anywhere}.code-block{break-inside:avoid}.equation{font-size:12pt;break-inside:avoid;padding:10px 14px}.equation>div{white-space:pre-wrap;min-width:0}.table-scroll{overflow:visible;border-radius:0}.table-scroll table{font-size:8pt;width:100%;table-layout:auto}.table-scroll th,.table-scroll td{padding:6px 7px;min-width:0}.table-scroll th{white-space:normal}thead{display:table-header-group}tr{break-inside:avoid}.report img{max-height:220mm;object-fit:contain;break-inside:avoid}.print-only{display:block}}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<a class="skip-link" href="#report">跳转至正文</a>
|
||||||
|
<header class="masthead">
|
||||||
|
<div class="masthead-inner">
|
||||||
|
<div class="eyebrow">ECONOMETRICS / WEEK 08</div>
|
||||||
|
<h1>第8周研究报告:回归自由度与财务、治理数据合并</h1>
|
||||||
|
<div class="masthead-bottom">
|
||||||
|
<div class="masthead-caption">计量经济学课程研究报告</div>
|
||||||
|
<div class="header-actions"><a href="Week8_研究报告.md">Markdown 文档</a><a href="week8_analysis.do">Stata 脚本</a><button type="button" onclick="window.print()">打印报告</button></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</header>
|
||||||
|
<div class="shell">
|
||||||
|
<aside class="sidebar" aria-label="报告目录">
|
||||||
|
<div class="sidebar-inner">
|
||||||
|
<div class="toc-title">内容导航</div>
|
||||||
|
<nav aria-label="章节"><ol class="toc"><li class="level-2"><a href="#一-研究问题与主要结论">一、研究问题与主要结论</a></li>
|
||||||
|
<li class="level-2"><a href="#二-为什么分母要减去-k-1">二、为什么分母要减去 k + 1</a></li>
|
||||||
|
<li class="level-3"><a href="#2-1-先区分随机误差和估计残差">2.1 先区分随机误差和估计残差</a></li>
|
||||||
|
<li class="level-3"><a href="#2-2-从正规方程理解自由度">2.2 从正规方程理解自由度</a></li>
|
||||||
|
<li class="level-3"><a href="#2-3-无偏性的矩阵推导">2.3 无偏性的矩阵推导</a></li>
|
||||||
|
<li class="level-3"><a href="#2-4-公式的适用边界">2.4 公式的适用边界</a></li>
|
||||||
|
<li class="level-2"><a href="#三-数据来源与变量口径">三、数据来源与变量口径</a></li>
|
||||||
|
<li class="level-2"><a href="#四-stata-数据整理与合并方法">四、Stata 数据整理与合并方法</a></li>
|
||||||
|
<li class="level-2"><a href="#五-合并结果与年度覆盖">五、合并结果与年度覆盖</a></li>
|
||||||
|
<li class="level-3"><a href="#5-1-全部年份的覆盖表">5.1 全部年份的覆盖表</a></li>
|
||||||
|
<li class="level-3"><a href="#5-2-2014年为何特别需要注意">5.2 2014年为何特别需要注意</a></li>
|
||||||
|
<li class="level-2"><a href="#六-缺失值-极值与描述统计">六、缺失值、极值与描述统计</a></li>
|
||||||
|
<li class="level-2"><a href="#七-用真实数据验证自由度公式">七、用真实数据验证自由度公式</a></li>
|
||||||
|
<li class="level-2"><a href="#八-复现步骤与交付文件">八、复现步骤与交付文件</a></li>
|
||||||
|
<li class="level-2"><a href="#九-结论与资料依据">九、结论与资料依据</a></li></ol></nav>
|
||||||
|
<div class="sidebar-note">正文、公式和样式均可离线阅读。关联数据与脚本保留在同一工作目录。</div>
|
||||||
|
</div>
|
||||||
|
</aside>
|
||||||
|
<main class="report" id="report">
|
||||||
|
<p>完成日期:2026年10月29日。软件:本机 Stata/SE 15.1,脚本使用 <code>version 15.0</code>。数据截止年份由文件内容确定,为2014年。</p>
|
||||||
|
<h2 id="一-研究问题与主要结论">一、研究问题与主要结论</h2>
|
||||||
|
<p>本次工作包括两个问题:解释题目图片中的误差方差估计为何使用分母 <code>n − k − 1</code>,以及将所给 RESSET 财务指标 ROA、ROE 与两份 CSMAR 治理综合信息文件整理、合并。<a href="1.png">题目原图</a>保留在工作目录。</p>
|
||||||
|
<p><strong>理论结论:</strong> 在含截距、设计矩阵满列秩的多元回归中,估计 k 个斜率及一个截距会对残差施加 k + 1 个独立线性约束。因此,残差自由度为 n − k − 1。在条件均值为零、同方差且误差间不相关的条件下,SSE 除以该自由度是误差方差的无偏估计。</p>
|
||||||
|
<p><strong>数据结论:</strong> 财务文件共有 34,897 条记录,覆盖1990—2014年;治理文件追加后共有 27,090 条记录,覆盖1999—2014年。按“证券代码文本+报告年份”一对一外连接,得到 <strong>39,416 条公司年度记录</strong>,其中 <strong>22,571 条两库匹配</strong>,匹配且 ROA、ROE 均非缺失的记录为 <strong>22,122 条</strong>。</p>
|
||||||
|
<blockquote>
|
||||||
|
<p>本报告的“所有年份”指所给文件中存在的全部年份。本次直接读取离线导出文件,没有重新登录数据库下载。2014年财务数据覆盖明显不足,文件中还存在非标准代码和非通常沪深A股代码前缀,不能将这些导出记录称为全市场全部上市公司的完整历史。</p>
|
||||||
|
</blockquote>
|
||||||
|
<h2 id="二-为什么分母要减去-k-1">二、为什么分母要减去 k + 1</h2>
|
||||||
|
<h3 id="2-1-先区分随机误差和估计残差">2.1 先区分随机误差和估计残差</h3>
|
||||||
|
<p>设总体回归模型为:</p>
|
||||||
|
<div aria-label="数学公式" class="equation" role="group"><div>yᵢ = β₀ + β₁xᵢ₁ + ⋯ + βₖxᵢₖ + uᵢ</div>
|
||||||
|
<div>eᵢ = yᵢ − ŷᵢ</div>
|
||||||
|
<div>SSE = ∑ᵢ₌₁ⁿ eᵢ²</div></div>
|
||||||
|
<p>其中,uᵢ 是不可直接观测的随机误差,eᵢ 是估计模型后得到的残差。n 是实际进入回归的完整观测数,k 是不含截距的解释变量个数。题目中的 σ̂² 估计的是随机误差方差,计算时使用可观测的残差平方和。</p>
|
||||||
|
<h3 id="2-2-从正规方程理解自由度">2.2 从正规方程理解自由度</h3>
|
||||||
|
<p>OLS 用同一批观测拟合 k + 1 个参数,使残差平方和尽可能小。一阶条件为:</p>
|
||||||
|
<div aria-label="数学公式" class="equation" role="group"><div>∑ᵢ₌₁ⁿ eᵢ = 0</div>
|
||||||
|
<div>∑ᵢ₌₁ⁿ xᵢⱼeᵢ = 0,j = 1, …, k</div></div>
|
||||||
|
<p>当设计矩阵满列秩时,这 k + 1 个条件是独立的线性约束。虽然残差向量有 n 个元素,它只能在 n − k − 1 维空间中变化。这就是残差自由度。这里指线性自由度,并不意味着任取 n − k − 1 个原始残差就一定在概率意义上相互独立。</p>
|
||||||
|
<p>只有截距时,估计的是样本均值,分母退化为 n − 1,与样本方差公式相同。简单线性回归估计截距和一个斜率,分母为 n − 2。</p>
|
||||||
|
<h3 id="2-3-无偏性的矩阵推导">2.3 无偏性的矩阵推导</h3>
|
||||||
|
<p>令 p = k + 1,X 包含截距列,且 rank(X) = p < n。假设条件均值 E(u∣X) = 0,条件方差矩阵 Var(u∣X) = σ²Iₙ。后一个条件同时要求同方差与不同观测误差不相关。</p>
|
||||||
|
<p>定义投影矩阵 H 与残差生成矩阵 M:</p>
|
||||||
|
<div aria-label="数学公式" class="equation" role="group"><div>y = Xβ + u</div>
|
||||||
|
<div>H = X(X′X)⁻¹X′,M = Iₙ − H</div>
|
||||||
|
<div>e = My = Mu</div>
|
||||||
|
<div>M′ = M,M² = M,tr(M) = n − p</div></div>
|
||||||
|
<p>利用 M 的对称性、幂等性与迹的性质:</p>
|
||||||
|
<div aria-label="数学公式" class="equation" role="group"><div>SSE = e′e = u′Mu</div>
|
||||||
|
<div>E(SSE∣X) = tr[M E(uu′∣X)]</div>
|
||||||
|
<div> = σ² tr(M)</div>
|
||||||
|
<div> = (n − k − 1)σ²</div></div>
|
||||||
|
<p>因此:</p>
|
||||||
|
<div aria-label="数学公式" class="equation" role="group"><div>σ̂² = SSE / (n − k − 1)</div>
|
||||||
|
<div>E(σ̂²∣X) = σ²</div></div>
|
||||||
|
<p>如果直接除以 n,则 E(SSE/n∣X) = [(n − k − 1)/n]σ²,存在向下偏误。自由度调整补偿了利用样本拟合参数所造成的样本内残差平方和下降。</p>
|
||||||
|
<p>例如 n = 100、k = 3、SSE = 192,残差自由度为96,无偏方差估计为192/96 = 2;直接除以100只得到1.92。</p>
|
||||||
|
<h3 id="2-4-公式的适用边界">2.4 公式的适用边界</h3>
|
||||||
|
<ul>
|
||||||
|
<li>一般公式是 <code>n − rank(X)</code>。无截距、完全共线性、虚拟变量和交互项等情况,应按实际独立参数数目计算。</li>
|
||||||
|
<li>正态性不是上述无偏性的必要条件。正态性用于进一步推导精确的卡方、t 和 F 分布。</li>
|
||||||
|
<li>若存在异方差或相关误差,记 Var(u∣X) = Ω,则 E(SSE∣X) = tr(MΩ),通常不能再写成 <code>(n − k − 1)σ²</code>。</li>
|
||||||
|
<li>自由度调整本身不能解决异方差、内生性或模型遗漏问题。采用企业聚类标准误时,Stata 返回的推断自由度可能以聚类数为基础,不能直接替代本题普通 OLS 的残差自由度。</li>
|
||||||
|
</ul>
|
||||||
|
<h2 id="三-数据来源与变量口径">三、数据来源与变量口径</h2>
|
||||||
|
<div class="table-scroll" tabindex="0"><table><thead>
|
||||||
|
<tr>
|
||||||
|
<th>输入文件</th>
|
||||||
|
<th>工作表</th>
|
||||||
|
<th>数据行数(不含说明行)</th>
|
||||||
|
<th>实际报告年份</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr>
|
||||||
|
<td><a href="data/FININDRATIO_OLD_5938AD12016_(%E9%94%90%E6%80%9D).xls">RESSET财务文件</a></td>
|
||||||
|
<td>FININDRATIO_OLD</td>
|
||||||
|
<td>34,897</td>
|
||||||
|
<td>1990—2014</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><a href="data/CG_Ybasic1999-2012.xls">CSMAR治理文件1999—2012</a></td>
|
||||||
|
<td>CG_Ybasic</td>
|
||||||
|
<td>21,902</td>
|
||||||
|
<td>1999—2012</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><a href="data/CG_Ybasic2013-2014.xls">CSMAR治理文件2013—2014</a></td>
|
||||||
|
<td>CG_Ybasic</td>
|
||||||
|
<td>5,188</td>
|
||||||
|
<td>2013—2014</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table></div>
|
||||||
|
<p>RESSET 文件共10列,第1行为中文标题;全部记录的 <code>Reporttype</code> 为 Q4,截止日期均为12月31日,<code>Lstflg</code> 全为 A,B/H 股票代码列全空。不同证券代码为3,633个。源文件中的“最新公司全称”保留为 <code>Lcomnm</code>,不将其视为逐年历史名称。</p>
|
||||||
|
<p>CSMAR 两份文件均为33列。第1行为英文字段名,第2行为中文含义,第3行为单位,第4行起才是数据。两份文件追加后包含2,737个不同证券代码。本次保留全部原始治理字段,而非仅保留示例回归使用的董事人数。</p>
|
||||||
|
<div class="table-scroll" tabindex="0"><table><thead>
|
||||||
|
<tr>
|
||||||
|
<th>分析字段</th>
|
||||||
|
<th>来源字段</th>
|
||||||
|
<th>处理与含义</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr>
|
||||||
|
<td><code>stkcd</code></td>
|
||||||
|
<td>RESSET <code>A_Stkcd</code>;CSMAR <code>Stkcd</code></td>
|
||||||
|
<td>作为文本,保留前导零,仅去除首尾空格;通常为六位</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>year</code></td>
|
||||||
|
<td>RESSET <code>Enddt</code>;CSMAR <code>Reptdt</code></td>
|
||||||
|
<td>使用报告截止年份,不使用公告年份</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>ROA</code></td>
|
||||||
|
<td>资产净利率(%)_ROA</td>
|
||||||
|
<td>保留原始数值和缺失值</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>ROE</code></td>
|
||||||
|
<td>净资产收益率(摊薄)(%)_ROE</td>
|
||||||
|
<td>保留摊薄口径的原始数值和缺失值</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>Y1101a</code></td>
|
||||||
|
<td>CSMAR 董事人数</td>
|
||||||
|
<td>原始数值字段,示例中构造 <code>board_size</code></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>Y1101b</code></td>
|
||||||
|
<td>CSMAR 独立董事人数</td>
|
||||||
|
<td>与董事人数之比构成 <code>independent_share</code></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>_merge</code></td>
|
||||||
|
<td>Stata <code>merge</code> 生成</td>
|
||||||
|
<td>1:仅财务;2:仅治理;3:两库匹配</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>complete_roa_roe</code></td>
|
||||||
|
<td>派生标记</td>
|
||||||
|
<td>两库匹配且ROA、ROE均非缺失时为1</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>code_format_exception</code></td>
|
||||||
|
<td>派生标记</td>
|
||||||
|
<td>原始证券代码非六位时为1</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table></div>
|
||||||
|
<p>完整字段含义和单位见 <a href="variable_dictionary.csv">变量字典</a>。<code>resset_excel_row</code>、<code>csmar_source</code> 和 <code>csmar_excel_row</code> 可追溯原始工作簿与行号。</p>
|
||||||
|
<p><strong>ROA、ROE 单位处理:</strong> 原列标题写有“(%)”,但 Excel 单元格格式为 General,未提供指标公式或完整数据库数据字典。仅凭标题、格式和数值分布,不能独立确认导出数值究竟采用百分数尺度还是比例小数尺度。本次不乘100、不除100,所有统计均以“原始数值尺度”报告,不将0.0498擅自解释为0.0498%或4.98%。</p>
|
||||||
|
<h2 id="四-stata-数据整理与合并方法">四、Stata 数据整理与合并方法</h2>
|
||||||
|
<ol>
|
||||||
|
<li>从RESSET工作表第2行读取数值,保存所有既有年份。验证Q4、12月31日及公司年度键唯一性,保留ROA、ROE原值。</li>
|
||||||
|
<li>分别导入两份CSMAR表,读取中文含义与单位后去掉两行说明;将人数、持股量、薪酬等字段转换为数值。未使用 <code>destring, force</code>,防止无提示地把异常文本变成缺失。</li>
|
||||||
|
<li>使用 <code>append</code> 将1999—2012和2013—2014治理数据纵向拼接,验证 <code>isid stkcd year</code>。</li>
|
||||||
|
<li>以财务数据为master、治理数据为using,执行 <code>merge 1:1 stkcd year</code>。保留三种合并状态,并核验匹配记录的完整截止日期相等。</li>
|
||||||
|
<li>保存完整外连接数据集,另存匹配子集;在汇总表中分别报告源数据缺失与对方来源未出现,不用零替代缺失。</li>
|
||||||
|
</ol>
|
||||||
|
<p>核心合并代码与完整脚本中的实现一致:</p>
|
||||||
|
<div class="code-block"><div class="code-label">stata</div><pre><code>use "resset_financial_all_years.dta", clear
|
||||||
|
isid stkcd year
|
||||||
|
merge 1:1 stkcd year using "csmar_governance_1999_2014.dta"
|
||||||
|
assert Enddt == csmar_date if _merge == 3
|
||||||
|
isid stkcd year
|
||||||
|
</code></pre></div>
|
||||||
|
<p>这里 <code>append</code> 增加治理表的年份记录,<code>merge</code> 按同一公司、同一报告年份横向增加变量。未使用 <code>merge m:m</code>,也未通过强制删除重复键来让合并继续;两端原始公司年度键本身均唯一。</p>
|
||||||
|
<p><strong>代码例外:</strong> RESSET 的Excel第14580、14581行,吉林通海高科技股份有限公司1998、1999年的股票代码原值均为文本 <code>0991</code>。CSMAR 中既没有 <code>0991</code>,也没有 <code>000991</code>。本次保留 <code>0991</code> 并标记,未推测补成另一证券代码,两条均保留在仅财务记录中。</p>
|
||||||
|
<h2 id="五-合并结果与年度覆盖">五、合并结果与年度覆盖</h2>
|
||||||
|
<div class="table-scroll" tabindex="0"><table><thead>
|
||||||
|
<tr>
|
||||||
|
<th>状态</th>
|
||||||
|
<th>条数</th>
|
||||||
|
<th>解释</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr>
|
||||||
|
<td><code>_merge = 1</code></td>
|
||||||
|
<td>12,326</td>
|
||||||
|
<td>财务表存在,对应公司年度不在治理表</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>_merge = 2</code></td>
|
||||||
|
<td>4,519</td>
|
||||||
|
<td>治理表存在,对应公司年度不在财务表</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>_merge = 3</code></td>
|
||||||
|
<td>22,571</td>
|
||||||
|
<td>同一证券代码、同一报告年份两库均存在</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>完整外连接</td>
|
||||||
|
<td>39,416</td>
|
||||||
|
<td>三类记录合计</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>匹配且双指标完整</td>
|
||||||
|
<td>22,122</td>
|
||||||
|
<td>是匹配记录的子集,不与前三类重复相加</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table></div>
|
||||||
|
<p>核对关系为:<code>34,897 + 27,090 − 22,571 = 39,416</code>。匹配率以财务记录为分母为 <strong>64.68%</strong>,以治理记录为分母为 <strong>83.32%</strong>。不同分母回答不同覆盖问题,不应混用。</p>
|
||||||
|
<p>仅财务的12,326条中,4,279条来自1990—1998年,早于治理文件起始年份;其余8,047条处于1999—2014年。仅治理的4,519条说明对应财务键未在所给文件中出现,不能据此判断公司当年没有财务报告。</p>
|
||||||
|
<p>两库匹配子集覆盖1999—2014年,共2,376个不同证券代码。该集合是非平衡公司年度面板,不能假定每家公司在每年都有完整记录。原始文件未提供足够的证券类别、历史上市状态及跨代码公司映射资料,本次未按公司名称或代码前缀强行归并实体。</p>
|
||||||
|
<h3 id="5-1-全部年份的覆盖表">5.1 全部年份的覆盖表</h3>
|
||||||
|
<div class="table-scroll" tabindex="0"><table><thead>
|
||||||
|
<tr>
|
||||||
|
<th>报告年</th>
|
||||||
|
<th>财务记录</th>
|
||||||
|
<th>治理记录</th>
|
||||||
|
<th>两库匹配</th>
|
||||||
|
<th>双指标完整</th>
|
||||||
|
<th>匹配/财务</th>
|
||||||
|
<th>匹配/治理</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr>
|
||||||
|
<td>1990</td>
|
||||||
|
<td>5</td>
|
||||||
|
<td>0</td>
|
||||||
|
<td>0</td>
|
||||||
|
<td>0</td>
|
||||||
|
<td>0.00%</td>
|
||||||
|
<td>—</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>1991</td>
|
||||||
|
<td>5</td>
|
||||||
|
<td>0</td>
|
||||||
|
<td>0</td>
|
||||||
|
<td>0</td>
|
||||||
|
<td>0.00%</td>
|
||||||
|
<td>—</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>1992</td>
|
||||||
|
<td>36</td>
|
||||||
|
<td>0</td>
|
||||||
|
<td>0</td>
|
||||||
|
<td>0</td>
|
||||||
|
<td>0.00%</td>
|
||||||
|
<td>—</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>1993</td>
|
||||||
|
<td>277</td>
|
||||||
|
<td>0</td>
|
||||||
|
<td>0</td>
|
||||||
|
<td>0</td>
|
||||||
|
<td>0.00%</td>
|
||||||
|
<td>—</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>1994</td>
|
||||||
|
<td>511</td>
|
||||||
|
<td>0</td>
|
||||||
|
<td>0</td>
|
||||||
|
<td>0</td>
|
||||||
|
<td>0.00%</td>
|
||||||
|
<td>—</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>1995</td>
|
||||||
|
<td>727</td>
|
||||||
|
<td>0</td>
|
||||||
|
<td>0</td>
|
||||||
|
<td>0</td>
|
||||||
|
<td>0.00%</td>
|
||||||
|
<td>—</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>1996</td>
|
||||||
|
<td>813</td>
|
||||||
|
<td>0</td>
|
||||||
|
<td>0</td>
|
||||||
|
<td>0</td>
|
||||||
|
<td>0.00%</td>
|
||||||
|
<td>—</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>1997</td>
|
||||||
|
<td>918</td>
|
||||||
|
<td>0</td>
|
||||||
|
<td>0</td>
|
||||||
|
<td>0</td>
|
||||||
|
<td>0.00%</td>
|
||||||
|
<td>—</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>1998</td>
|
||||||
|
<td>987</td>
|
||||||
|
<td>0</td>
|
||||||
|
<td>0</td>
|
||||||
|
<td>0</td>
|
||||||
|
<td>0.00%</td>
|
||||||
|
<td>—</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>1999</td>
|
||||||
|
<td>1,088</td>
|
||||||
|
<td>949</td>
|
||||||
|
<td>782</td>
|
||||||
|
<td>776</td>
|
||||||
|
<td>71.88%</td>
|
||||||
|
<td>82.40%</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>2000</td>
|
||||||
|
<td>1,161</td>
|
||||||
|
<td>1,108</td>
|
||||||
|
<td>924</td>
|
||||||
|
<td>918</td>
|
||||||
|
<td>79.59%</td>
|
||||||
|
<td>83.39%</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>2001</td>
|
||||||
|
<td>1,258</td>
|
||||||
|
<td>1,163</td>
|
||||||
|
<td>977</td>
|
||||||
|
<td>969</td>
|
||||||
|
<td>77.66%</td>
|
||||||
|
<td>84.01%</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>2002</td>
|
||||||
|
<td>1,270</td>
|
||||||
|
<td>1,229</td>
|
||||||
|
<td>1,045</td>
|
||||||
|
<td>1,032</td>
|
||||||
|
<td>82.28%</td>
|
||||||
|
<td>85.03%</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>2003</td>
|
||||||
|
<td>1,354</td>
|
||||||
|
<td>1,288</td>
|
||||||
|
<td>1,107</td>
|
||||||
|
<td>1,090</td>
|
||||||
|
<td>81.76%</td>
|
||||||
|
<td>85.95%</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>2004</td>
|
||||||
|
<td>1,470</td>
|
||||||
|
<td>1,378</td>
|
||||||
|
<td>1,204</td>
|
||||||
|
<td>1,177</td>
|
||||||
|
<td>81.90%</td>
|
||||||
|
<td>87.37%</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>2005</td>
|
||||||
|
<td>1,522</td>
|
||||||
|
<td>1,375</td>
|
||||||
|
<td>1,217</td>
|
||||||
|
<td>1,169</td>
|
||||||
|
<td>79.96%</td>
|
||||||
|
<td>88.51%</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>2006</td>
|
||||||
|
<td>1,690</td>
|
||||||
|
<td>1,456</td>
|
||||||
|
<td>1,292</td>
|
||||||
|
<td>1,235</td>
|
||||||
|
<td>76.45%</td>
|
||||||
|
<td>88.74%</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>2007</td>
|
||||||
|
<td>2,051</td>
|
||||||
|
<td>1,572</td>
|
||||||
|
<td>1,396</td>
|
||||||
|
<td>1,343</td>
|
||||||
|
<td>68.06%</td>
|
||||||
|
<td>88.80%</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>2008</td>
|
||||||
|
<td>2,292</td>
|
||||||
|
<td>1,626</td>
|
||||||
|
<td>1,447</td>
|
||||||
|
<td>1,401</td>
|
||||||
|
<td>63.13%</td>
|
||||||
|
<td>88.99%</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>2009</td>
|
||||||
|
<td>2,441</td>
|
||||||
|
<td>1,774</td>
|
||||||
|
<td>1,593</td>
|
||||||
|
<td>1,539</td>
|
||||||
|
<td>65.26%</td>
|
||||||
|
<td>89.80%</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>2010</td>
|
||||||
|
<td>2,580</td>
|
||||||
|
<td>2,129</td>
|
||||||
|
<td>1,941</td>
|
||||||
|
<td>1,896</td>
|
||||||
|
<td>75.23%</td>
|
||||||
|
<td>91.17%</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>2011</td>
|
||||||
|
<td>3,209</td>
|
||||||
|
<td>2,363</td>
|
||||||
|
<td>2,171</td>
|
||||||
|
<td>2,135</td>
|
||||||
|
<td>67.65%</td>
|
||||||
|
<td>91.87%</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>2012</td>
|
||||||
|
<td>3,554</td>
|
||||||
|
<td>2,492</td>
|
||||||
|
<td>2,297</td>
|
||||||
|
<td>2,276</td>
|
||||||
|
<td>64.63%</td>
|
||||||
|
<td>92.17%</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>2013</td>
|
||||||
|
<td>2,841</td>
|
||||||
|
<td>2,536</td>
|
||||||
|
<td>2,342</td>
|
||||||
|
<td>2,338</td>
|
||||||
|
<td>82.44%</td>
|
||||||
|
<td>92.35%</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>2014</td>
|
||||||
|
<td>837</td>
|
||||||
|
<td>2,652</td>
|
||||||
|
<td>836</td>
|
||||||
|
<td>828</td>
|
||||||
|
<td>99.88%</td>
|
||||||
|
<td>31.52%</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table></div>
|
||||||
|
<p>“双指标完整”指该年两库匹配且ROA、ROE均非缺失;“—”表示治理文件该年没有记录,分母为零。完整审计表还包括各类未匹配及财务源内缺失数,见 <a href="merge_by_year.csv">逐年合并统计</a>。</p>
|
||||||
|
<p><img alt="各报告年度的财务、治理及匹配记录数" src="annual_coverage.png"/></p>
|
||||||
|
<p>图中 RESSET 表示财务文件记录数,CSMAR 表示治理文件记录数,Matched 表示两库匹配数。该图描述文件覆盖,不能解释为市场公司数量或企业退出趋势。</p>
|
||||||
|
<h3 id="5-2-2014年为何特别需要注意">5.2 2014年为何特别需要注意</h3>
|
||||||
|
<p>2014年财务记录仅837条,治理记录为2,652条,匹配836条。对财务表计算的匹配率为99.88%,但对治理表计算仅为31.52%。较高的前一个比例不能证明2014年财务数据覆盖完整。该年财务文件仅出现000、001、002和600前缀;2013年有2,841条财务记录,2014年的明显减少应首先视为导出覆盖问题,不能直接推断真实经济变化。</p>
|
||||||
|
<h2 id="六-缺失值-极值与描述统计">六、缺失值、极值与描述统计</h2>
|
||||||
|
<p>全部财务记录中,ROA缺失69条、ROE缺失726条。匹配后,ROA无缺失,ROE仍缺失449条,故匹配与“双指标完整”是两个不同筛选条件。完整外连接中仅治理记录的ROA/ROE为空属于来源缺失,不应与财务表内部缺失混为一谈。</p>
|
||||||
|
<p>以下数值均使用原始尺度,未进行百分数换算。各变量按自身非缺失观测计算;“匹配且双指标完整”使用共同的22,122条记录。</p>
|
||||||
|
<div class="table-scroll" tabindex="0"><table><thead>
|
||||||
|
<tr>
|
||||||
|
<th>样本</th>
|
||||||
|
<th>变量</th>
|
||||||
|
<th>有效数</th>
|
||||||
|
<th>缺失数</th>
|
||||||
|
<th>均值</th>
|
||||||
|
<th>中位数</th>
|
||||||
|
<th>标准差</th>
|
||||||
|
<th>最小值</th>
|
||||||
|
<th>最大值</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr>
|
||||||
|
<td>全部财务</td>
|
||||||
|
<td>ROA</td>
|
||||||
|
<td>34,828</td>
|
||||||
|
<td>69</td>
|
||||||
|
<td>-0.00162996</td>
|
||||||
|
<td>0.0498</td>
|
||||||
|
<td>10.471</td>
|
||||||
|
<td>-1,950.73</td>
|
||||||
|
<td>64.7546</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>全部财务</td>
|
||||||
|
<td>ROE</td>
|
||||||
|
<td>34,171</td>
|
||||||
|
<td>726</td>
|
||||||
|
<td>35.2343</td>
|
||||||
|
<td>0.0958</td>
|
||||||
|
<td>6,512.09</td>
|
||||||
|
<td>-1,950.73</td>
|
||||||
|
<td>1.20378e+06</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>两库匹配</td>
|
||||||
|
<td>ROA</td>
|
||||||
|
<td>22,571</td>
|
||||||
|
<td>0</td>
|
||||||
|
<td>0.0360606</td>
|
||||||
|
<td>0.0349</td>
|
||||||
|
<td>0.639555</td>
|
||||||
|
<td>-64.8192</td>
|
||||||
|
<td>64.7546</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>两库匹配</td>
|
||||||
|
<td>ROE</td>
|
||||||
|
<td>22,122</td>
|
||||||
|
<td>449</td>
|
||||||
|
<td>0.019237</td>
|
||||||
|
<td>0.0691</td>
|
||||||
|
<td>3.06854</td>
|
||||||
|
<td>-188.392</td>
|
||||||
|
<td>159.903</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>匹配且双指标完整</td>
|
||||||
|
<td>ROA</td>
|
||||||
|
<td>22,122</td>
|
||||||
|
<td>0</td>
|
||||||
|
<td>0.0403534</td>
|
||||||
|
<td>0.0355</td>
|
||||||
|
<td>0.106527</td>
|
||||||
|
<td>-1.1943</td>
|
||||||
|
<td>7.1963</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>匹配且双指标完整</td>
|
||||||
|
<td>ROE</td>
|
||||||
|
<td>22,122</td>
|
||||||
|
<td>0</td>
|
||||||
|
<td>0.019237</td>
|
||||||
|
<td>0.0691</td>
|
||||||
|
<td>3.06854</td>
|
||||||
|
<td>-188.392</td>
|
||||||
|
<td>159.903</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table></div>
|
||||||
|
<p>表中数字为便于阅读而四舍五入;完整精度及P1、P25、P75、P99见 <a href="descriptive_statistics.csv">描述统计文件</a>。分位点采用Stata <code>summarize, detail</code> 的定义,可能与采用线性插值的其他软件不同。</p>
|
||||||
|
<p>源文件含明显极端观测,例如证券代码400033在2012年的ROA为−1,950.7321,代码430082在2011年的ROE为1,203,783.875。全部财务ROE均值约35.2343,而中位数为0.0958,说明均值受尾部值显著影响。匹配ROA的中位数为0.0349,匹配ROE的中位数为0.0691;这些只是导出样本的分布描述,不构成总体盈利水平的代表性估计。</p>
|
||||||
|
<p>本次没有缩尾、截尾、删除零值或替换异常值,因为题目要求提取与合并原始指标,且没有提供修订依据。极值记录见 <a href="source_extremes.csv">原始极值清单</a>。后续实证分析若需要异常值处理,应先核验原始财务报表和指标定义,并单独报告处理规则及敏感性结果。</p>
|
||||||
|
<h2 id="七-用真实数据验证自由度公式">七、用真实数据验证自由度公式</h2>
|
||||||
|
<p>为将第一题与Stata输出对应,选取2013年两库匹配的横截面,使用ROA作为因变量、董事人数与独立董事比例作为两个解释变量。董事人数须为正,独董比例须在0到1之间。该年2,342条匹配记录全部满足条件,无额外剔除。</p>
|
||||||
|
<div aria-label="数学公式" class="equation" role="group"><div>ROAᵢ = β₀ + β₁board_sizeᵢ + β₂independent_shareᵢ + uᵢ</div>
|
||||||
|
<div>independent_shareᵢ = 独立董事人数ᵢ / 董事人数ᵢ</div></div>
|
||||||
|
<p>使用默认普通OLS,仅验证残差自由度与MSE的计算,不将该模型作为治理影响盈利能力的因果识别模型,也不据此断言其误差满足同方差假设。</p>
|
||||||
|
<div class="code-block"><div class="code-label">stata</div><pre><code>regress ROA board_size independent_share if matched & year == 2013 & valid_board
|
||||||
|
display e(N)
|
||||||
|
display e(df_r)
|
||||||
|
display e(rss)/e(df_r)
|
||||||
|
display e(rmse)^2
|
||||||
|
</code></pre></div>
|
||||||
|
<div class="table-scroll" tabindex="0"><table><thead>
|
||||||
|
<tr>
|
||||||
|
<th>项目</th>
|
||||||
|
<th>Stata 实际结果</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr>
|
||||||
|
<td>实际估计样本 n</td>
|
||||||
|
<td>2,342</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>斜率个数 k</td>
|
||||||
|
<td>2</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>包括截距的参数个数</td>
|
||||||
|
<td>3</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>残差自由度 n − k − 1</td>
|
||||||
|
<td>2,339</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>SSE = e(rss)</td>
|
||||||
|
<td>65.691748161464</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>SSE / (n − k − 1)</td>
|
||||||
|
<td>0.028085398957</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>e(rmse)²</td>
|
||||||
|
<td>0.028085398957</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>SSE / n</td>
|
||||||
|
<td>0.028049422785</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table></div>
|
||||||
|
<p>因此该例恰好满足:</p>
|
||||||
|
<div aria-label="数学公式" class="equation" role="group"><div>n − k − 1 = 2342 − 2 − 1 = 2339</div>
|
||||||
|
<div>σ̂² = 65.691748161464 / 2339 = 0.028085398957</div></div>
|
||||||
|
<p>脚本同时用预测残差重新计算SSE,结果与 <code>e(rss)</code> 一致。残差之和为3.718e-15,董事人数与残差乘积之和为-3.256e-13,独董比例与残差乘积之和为-1.215e-15,均在浮点计算误差范围内接近零,验证了相应正规方程约束。</p>
|
||||||
|
<p><code>e(rmse)</code> 是误差标准差的估计,其平方才是本题中的误差方差估计。回归的完整数值输出见 <a href="week8_stata.log">Stata运行日志</a>,保存的估计结果为 <a href="df_example_ols.ster">df_example_ols.ster</a>,逐项验证数值为 <a href="df_example_metrics.csv">df_example_metrics.csv</a>。</p>
|
||||||
|
<h2 id="八-复现步骤与交付文件">八、复现步骤与交付文件</h2>
|
||||||
|
<p>在Stata命令窗口运行:</p>
|
||||||
|
<div class="code-block"><div class="code-label">stata</div><pre><code>do "C:/Users/cattom/Documents/econometrics/Week8/week8_analysis.do"
|
||||||
|
</code></pre></div>
|
||||||
|
<p>脚本直接读取原始Excel,完成整理、合并、审计、描述统计、图表和OLS示例。无需安装额外Stata命令。重复运行会替换本次生成的同名输出,原始 <code>data</code> 目录不被修改。若移动目录,只需修改do-file开头的 <code>cd</code> 行。</p>
|
||||||
|
<p>也可从PowerShell调用指定Stata程序:</p>
|
||||||
|
<div class="code-block"><div class="code-label">powershell</div><pre><code>Start-Process -FilePath 'D:\Stata15\StataSE-64.exe' `
|
||||||
|
-ArgumentList @('/e','do','C:\Users\cattom\Documents\econometrics\Week8\week8_analysis.do') `
|
||||||
|
-WorkingDirectory 'C:\Users\cattom\Documents\econometrics\Week8' `
|
||||||
|
-WindowStyle Hidden -Wait
|
||||||
|
</code></pre></div>
|
||||||
|
<p>批处理自动产生 <code>week8_analysis.log</code>,研究日志为 <code>week8_stata.log</code>。成功结束标识为 <code>WEEK8_ANALYSIS_COMPLETED_SUCCESSFULLY</code>。</p>
|
||||||
|
<div class="table-scroll" tabindex="0"><table><thead>
|
||||||
|
<tr>
|
||||||
|
<th>文件</th>
|
||||||
|
<th>用途</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr>
|
||||||
|
<td><a href="Week8_%E7%A0%94%E7%A9%B6%E6%8A%A5%E5%91%8A.md">Week8_研究报告.md</a></td>
|
||||||
|
<td>研究过程、推导与结论</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><a href="Week8_%E7%A0%94%E7%A9%B6%E6%8A%A5%E5%91%8A.html">Week8_研究报告.html</a></td>
|
||||||
|
<td>可离线打开、带导航和打印样式的HTML5页面</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><a href="financial_governance_all_years.dta">financial_governance_all_years.dta</a></td>
|
||||||
|
<td>主交付数据,39,416条,保留全部年份与未匹配记录</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><a href="financial_governance_all_years.csv">financial_governance_all_years.csv</a></td>
|
||||||
|
<td>同一主数据的UTF-8通用交换版本</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><a href="financial_governance_matched.dta">financial_governance_matched.dta</a></td>
|
||||||
|
<td>两库匹配数据,22,571条,保留ROE缺失记录</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><a href="resset_financial_all_years.dta">resset_financial_all_years.dta</a></td>
|
||||||
|
<td>整理后的全部34,897条财务记录</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><a href="csmar_governance_1999_2014.dta">csmar_governance_1999_2014.dta</a></td>
|
||||||
|
<td>追加后的27,090条治理记录</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><a href="week8_analysis.do">week8_analysis.do</a>;<a href="week8_stata.log">week8_stata.log</a></td>
|
||||||
|
<td>可复现Stata脚本与完整研究运行日志</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><a href="merge_by_year.csv">merge_by_year.csv</a>;<a href="processing_audit.csv">processing_audit.csv</a></td>
|
||||||
|
<td>年度覆盖及处理过程计数</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><a href="unmatched_records.csv">unmatched_records.csv</a>;<a href="source_extremes.csv">source_extremes.csv</a></td>
|
||||||
|
<td>未匹配记录与原始极值清单</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><a href="variable_dictionary.csv">variable_dictionary.csv</a></td>
|
||||||
|
<td>字段、中文含义、单位和来源</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><a href="descriptive_statistics.csv">descriptive_statistics.csv</a></td>
|
||||||
|
<td>各样本的有效数、缺失与分布统计</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><a href="annual_coverage.png">annual_coverage.png</a>;<a href="annual_coverage.gph">annual_coverage.gph</a></td>
|
||||||
|
<td>Stata生成的年度覆盖图与可编辑图形</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><a href="df_example_ols.ster">df_example_ols.ster</a>;<a href="df_example_sample.dta">df_example_sample.dta</a></td>
|
||||||
|
<td>OLS示例估计结果、样本与残差</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><a href="df_example_metrics.csv">df_example_metrics.csv</a></td>
|
||||||
|
<td>残差自由度、SSE、MSE等核验值</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><a href="validation_results.json">validation_results.json</a></td>
|
||||||
|
<td>独立复核记录</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table></div>
|
||||||
|
<p>优先使用DTA以保留证券代码字符串、日期、变量标签和来源标记。CSV导入其他软件时应把 <code>stkcd</code> 作为文本读取,避免丢失前导零;CSMAR备注含换行,须用标准CSV解析器读取,不能按文本行数统计观测。</p>
|
||||||
|
<p>只需要同时拥有财务与治理资料时,可从主数据 <code>keep if _merge == 3</code>;同时要求两个财务指标非缺失时,再加 <code>keep if complete_roa_roe == 1</code>。这两个筛选产生22,571和22,122两个不同样本,不应混称。</p>
|
||||||
|
<p>Stata 的键唯一性、报告日期一致性及残差自由度断言均已通过。独立核验共170项检查全部通过,其中原始Excel与整理、合并DTA比较了2,485,880个源单元格;数值、缺失值和中文文本均与来源一致,最长3,627字符的备注完整保留。主CSV的55列与DTA逐行一致,12项回归指标在CSV和DTA中也一致。CSV导出时对非日期数值使用完整双精度格式,避免默认显示格式造成大额数值四舍五入。源文件SHA-256及详细检查结果见 <code>validation_results.json</code>。</p>
|
||||||
|
<p>报告由 <code>build_report.py</code> 读取实际Stata汇总结果生成,<code>build_html.py</code> 将Markdown渲染成离线HTML;这些辅助脚本不参与替代Stata的数据合并与估计。若数据更换,先重跑Stata,再运行独立核验和两份报告构建脚本。</p>
|
||||||
|
<h2 id="九-结论与资料依据">九、结论与资料依据</h2>
|
||||||
|
<p>自由度调整的核心是:样本被用于估计参数后,残差可自由变化的维数随之减少。在本题的经典条件下,SSE的期望为 <code>(n − k − 1)σ²</code>,所以应除以残差自由度。2013年示例回归的残差自由度为2,339,SSE除以该值与Stata的Root MSE平方一致。</p>
|
||||||
|
<p>数据任务已形成保留全部现有年份的可追溯主数据及匹配子集。主要研究限制来自源文件覆盖、ROA/ROE经济尺度尚未获得数据字典确认,以及极值和缺失。现有结果支持描述给定文件的覆盖与数值分布,尚不足以作全市场推断或公司治理的因果结论。</p>
|
||||||
|
<p>资料依据包括:用户提供的题目图片与三份原始Excel;原始工作簿字段行和单位行;本机Stata 15官方帮助文档 <code>D:/Stata15/ado/base/r/regress.sthlp</code>(Stored results 中对 <code>e(N)</code>、<code>e(df_m)</code>、<code>e(rss)</code>、<code>e(df_r)</code>、<code>e(rmse)</code> 的定义);本机 <code>help merge</code> 对1:1合并及 <code>_merge</code> 状态的说明;以及本次实际运行的Stata日志。理论部分由正规方程和投影矩阵性质逐步推导。本次未成功取得在线手册正文,故不引用未经核验的网页页码或数据库指标公式。</p>
|
||||||
|
</main>
|
||||||
|
</div>
|
||||||
|
<footer class="footer">第8周 · 计量经济学</footer>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
@@ -0,0 +1,304 @@
|
|||||||
|
# 第8周研究报告:回归自由度与财务、治理数据合并
|
||||||
|
|
||||||
|
完成日期:2026年10月29日。软件:本机 Stata/SE 15.1,脚本使用 `version 15.0`。数据截止年份由文件内容确定,为2014年。
|
||||||
|
|
||||||
|
## 一、研究问题与主要结论
|
||||||
|
|
||||||
|
本次工作包括两个问题:解释题目图片中的误差方差估计为何使用分母 `n − k − 1`,以及将所给 RESSET 财务指标 ROA、ROE 与两份 CSMAR 治理综合信息文件整理、合并。[题目原图](1.png)保留在工作目录。
|
||||||
|
|
||||||
|
**理论结论:** 在含截距、设计矩阵满列秩的多元回归中,估计 k 个斜率及一个截距会对残差施加 k + 1 个独立线性约束。因此,残差自由度为 n − k − 1。在条件均值为零、同方差且误差间不相关的条件下,SSE 除以该自由度是误差方差的无偏估计。
|
||||||
|
|
||||||
|
**数据结论:** 财务文件共有 34,897 条记录,覆盖1990—2014年;治理文件追加后共有 27,090 条记录,覆盖1999—2014年。按“证券代码文本+报告年份”一对一外连接,得到 **39,416 条公司年度记录**,其中 **22,571 条两库匹配**,匹配且 ROA、ROE 均非缺失的记录为 **22,122 条**。
|
||||||
|
|
||||||
|
> 本报告的“所有年份”指所给文件中存在的全部年份。本次直接读取离线导出文件,没有重新登录数据库下载。2014年财务数据覆盖明显不足,文件中还存在非标准代码和非通常沪深A股代码前缀,不能将这些导出记录称为全市场全部上市公司的完整历史。
|
||||||
|
|
||||||
|
## 二、为什么分母要减去 k + 1
|
||||||
|
|
||||||
|
### 2.1 先区分随机误差和估计残差
|
||||||
|
|
||||||
|
设总体回归模型为:
|
||||||
|
|
||||||
|
```math
|
||||||
|
yᵢ = β₀ + β₁xᵢ₁ + ⋯ + βₖxᵢₖ + uᵢ
|
||||||
|
eᵢ = yᵢ − ŷᵢ
|
||||||
|
SSE = ∑ᵢ₌₁ⁿ eᵢ²
|
||||||
|
```
|
||||||
|
|
||||||
|
其中,uᵢ 是不可直接观测的随机误差,eᵢ 是估计模型后得到的残差。n 是实际进入回归的完整观测数,k 是不含截距的解释变量个数。题目中的 σ̂² 估计的是随机误差方差,计算时使用可观测的残差平方和。
|
||||||
|
|
||||||
|
### 2.2 从正规方程理解自由度
|
||||||
|
|
||||||
|
OLS 用同一批观测拟合 k + 1 个参数,使残差平方和尽可能小。一阶条件为:
|
||||||
|
|
||||||
|
```math
|
||||||
|
∑ᵢ₌₁ⁿ eᵢ = 0
|
||||||
|
∑ᵢ₌₁ⁿ xᵢⱼeᵢ = 0,j = 1, …, k
|
||||||
|
```
|
||||||
|
|
||||||
|
当设计矩阵满列秩时,这 k + 1 个条件是独立的线性约束。虽然残差向量有 n 个元素,它只能在 n − k − 1 维空间中变化。这就是残差自由度。这里指线性自由度,并不意味着任取 n − k − 1 个原始残差就一定在概率意义上相互独立。
|
||||||
|
|
||||||
|
只有截距时,估计的是样本均值,分母退化为 n − 1,与样本方差公式相同。简单线性回归估计截距和一个斜率,分母为 n − 2。
|
||||||
|
|
||||||
|
### 2.3 无偏性的矩阵推导
|
||||||
|
|
||||||
|
令 p = k + 1,X 包含截距列,且 rank(X) = p < n。假设条件均值 E(u∣X) = 0,条件方差矩阵 Var(u∣X) = σ²Iₙ。后一个条件同时要求同方差与不同观测误差不相关。
|
||||||
|
|
||||||
|
定义投影矩阵 H 与残差生成矩阵 M:
|
||||||
|
|
||||||
|
```math
|
||||||
|
y = Xβ + u
|
||||||
|
H = X(X′X)⁻¹X′,M = Iₙ − H
|
||||||
|
e = My = Mu
|
||||||
|
M′ = M,M² = M,tr(M) = n − p
|
||||||
|
```
|
||||||
|
|
||||||
|
利用 M 的对称性、幂等性与迹的性质:
|
||||||
|
|
||||||
|
```math
|
||||||
|
SSE = e′e = u′Mu
|
||||||
|
E(SSE∣X) = tr[M E(uu′∣X)]
|
||||||
|
= σ² tr(M)
|
||||||
|
= (n − k − 1)σ²
|
||||||
|
```
|
||||||
|
|
||||||
|
因此:
|
||||||
|
|
||||||
|
```math
|
||||||
|
σ̂² = SSE / (n − k − 1)
|
||||||
|
E(σ̂²∣X) = σ²
|
||||||
|
```
|
||||||
|
|
||||||
|
如果直接除以 n,则 E(SSE/n∣X) = [(n − k − 1)/n]σ²,存在向下偏误。自由度调整补偿了利用样本拟合参数所造成的样本内残差平方和下降。
|
||||||
|
|
||||||
|
例如 n = 100、k = 3、SSE = 192,残差自由度为96,无偏方差估计为192/96 = 2;直接除以100只得到1.92。
|
||||||
|
|
||||||
|
### 2.4 公式的适用边界
|
||||||
|
|
||||||
|
- 一般公式是 `n − rank(X)`。无截距、完全共线性、虚拟变量和交互项等情况,应按实际独立参数数目计算。
|
||||||
|
- 正态性不是上述无偏性的必要条件。正态性用于进一步推导精确的卡方、t 和 F 分布。
|
||||||
|
- 若存在异方差或相关误差,记 Var(u∣X) = Ω,则 E(SSE∣X) = tr(MΩ),通常不能再写成 `(n − k − 1)σ²`。
|
||||||
|
- 自由度调整本身不能解决异方差、内生性或模型遗漏问题。采用企业聚类标准误时,Stata 返回的推断自由度可能以聚类数为基础,不能直接替代本题普通 OLS 的残差自由度。
|
||||||
|
|
||||||
|
## 三、数据来源与变量口径
|
||||||
|
|
||||||
|
| 输入文件 | 工作表 | 数据行数(不含说明行) | 实际报告年份 |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| [RESSET财务文件](<data/FININDRATIO_OLD_5938AD12016_(锐思).xls>) | FININDRATIO_OLD | 34,897 | 1990—2014 |
|
||||||
|
| [CSMAR治理文件1999—2012](data/CG_Ybasic1999-2012.xls) | CG_Ybasic | 21,902 | 1999—2012 |
|
||||||
|
| [CSMAR治理文件2013—2014](data/CG_Ybasic2013-2014.xls) | CG_Ybasic | 5,188 | 2013—2014 |
|
||||||
|
|
||||||
|
RESSET 文件共10列,第1行为中文标题;全部记录的 `Reporttype` 为 Q4,截止日期均为12月31日,`Lstflg` 全为 A,B/H 股票代码列全空。不同证券代码为3,633个。源文件中的“最新公司全称”保留为 `Lcomnm`,不将其视为逐年历史名称。
|
||||||
|
|
||||||
|
CSMAR 两份文件均为33列。第1行为英文字段名,第2行为中文含义,第3行为单位,第4行起才是数据。两份文件追加后包含2,737个不同证券代码。本次保留全部原始治理字段,而非仅保留示例回归使用的董事人数。
|
||||||
|
|
||||||
|
| 分析字段 | 来源字段 | 处理与含义 |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `stkcd` | RESSET `A_Stkcd`;CSMAR `Stkcd` | 作为文本,保留前导零,仅去除首尾空格;通常为六位 |
|
||||||
|
| `year` | RESSET `Enddt`;CSMAR `Reptdt` | 使用报告截止年份,不使用公告年份 |
|
||||||
|
| `ROA` | 资产净利率(%)_ROA | 保留原始数值和缺失值 |
|
||||||
|
| `ROE` | 净资产收益率(摊薄)(%)_ROE | 保留摊薄口径的原始数值和缺失值 |
|
||||||
|
| `Y1101a` | CSMAR 董事人数 | 原始数值字段,示例中构造 `board_size` |
|
||||||
|
| `Y1101b` | CSMAR 独立董事人数 | 与董事人数之比构成 `independent_share` |
|
||||||
|
| `_merge` | Stata `merge` 生成 | 1:仅财务;2:仅治理;3:两库匹配 |
|
||||||
|
| `complete_roa_roe` | 派生标记 | 两库匹配且ROA、ROE均非缺失时为1 |
|
||||||
|
| `code_format_exception` | 派生标记 | 原始证券代码非六位时为1 |
|
||||||
|
|
||||||
|
完整字段含义和单位见 [变量字典](variable_dictionary.csv)。`resset_excel_row`、`csmar_source` 和 `csmar_excel_row` 可追溯原始工作簿与行号。
|
||||||
|
|
||||||
|
**ROA、ROE 单位处理:** 原列标题写有“(%)”,但 Excel 单元格格式为 General,未提供指标公式或完整数据库数据字典。仅凭标题、格式和数值分布,不能独立确认导出数值究竟采用百分数尺度还是比例小数尺度。本次不乘100、不除100,所有统计均以“原始数值尺度”报告,不将0.0498擅自解释为0.0498%或4.98%。
|
||||||
|
|
||||||
|
## 四、Stata 数据整理与合并方法
|
||||||
|
|
||||||
|
1. 从RESSET工作表第2行读取数值,保存所有既有年份。验证Q4、12月31日及公司年度键唯一性,保留ROA、ROE原值。
|
||||||
|
2. 分别导入两份CSMAR表,读取中文含义与单位后去掉两行说明;将人数、持股量、薪酬等字段转换为数值。未使用 `destring, force`,防止无提示地把异常文本变成缺失。
|
||||||
|
3. 使用 `append` 将1999—2012和2013—2014治理数据纵向拼接,验证 `isid stkcd year`。
|
||||||
|
4. 以财务数据为master、治理数据为using,执行 `merge 1:1 stkcd year`。保留三种合并状态,并核验匹配记录的完整截止日期相等。
|
||||||
|
5. 保存完整外连接数据集,另存匹配子集;在汇总表中分别报告源数据缺失与对方来源未出现,不用零替代缺失。
|
||||||
|
|
||||||
|
核心合并代码与完整脚本中的实现一致:
|
||||||
|
|
||||||
|
```stata
|
||||||
|
use "resset_financial_all_years.dta", clear
|
||||||
|
isid stkcd year
|
||||||
|
merge 1:1 stkcd year using "csmar_governance_1999_2014.dta"
|
||||||
|
assert Enddt == csmar_date if _merge == 3
|
||||||
|
isid stkcd year
|
||||||
|
```
|
||||||
|
|
||||||
|
这里 `append` 增加治理表的年份记录,`merge` 按同一公司、同一报告年份横向增加变量。未使用 `merge m:m`,也未通过强制删除重复键来让合并继续;两端原始公司年度键本身均唯一。
|
||||||
|
|
||||||
|
**代码例外:** RESSET 的Excel第14580、14581行,吉林通海高科技股份有限公司1998、1999年的股票代码原值均为文本 `0991`。CSMAR 中既没有 `0991`,也没有 `000991`。本次保留 `0991` 并标记,未推测补成另一证券代码,两条均保留在仅财务记录中。
|
||||||
|
|
||||||
|
## 五、合并结果与年度覆盖
|
||||||
|
|
||||||
|
| 状态 | 条数 | 解释 |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `_merge = 1` | 12,326 | 财务表存在,对应公司年度不在治理表 |
|
||||||
|
| `_merge = 2` | 4,519 | 治理表存在,对应公司年度不在财务表 |
|
||||||
|
| `_merge = 3` | 22,571 | 同一证券代码、同一报告年份两库均存在 |
|
||||||
|
| 完整外连接 | 39,416 | 三类记录合计 |
|
||||||
|
| 匹配且双指标完整 | 22,122 | 是匹配记录的子集,不与前三类重复相加 |
|
||||||
|
|
||||||
|
核对关系为:`34,897 + 27,090 − 22,571 = 39,416`。匹配率以财务记录为分母为 **64.68%**,以治理记录为分母为 **83.32%**。不同分母回答不同覆盖问题,不应混用。
|
||||||
|
|
||||||
|
仅财务的12,326条中,4,279条来自1990—1998年,早于治理文件起始年份;其余8,047条处于1999—2014年。仅治理的4,519条说明对应财务键未在所给文件中出现,不能据此判断公司当年没有财务报告。
|
||||||
|
|
||||||
|
两库匹配子集覆盖1999—2014年,共2,376个不同证券代码。该集合是非平衡公司年度面板,不能假定每家公司在每年都有完整记录。原始文件未提供足够的证券类别、历史上市状态及跨代码公司映射资料,本次未按公司名称或代码前缀强行归并实体。
|
||||||
|
|
||||||
|
### 5.1 全部年份的覆盖表
|
||||||
|
|
||||||
|
| 报告年 | 财务记录 | 治理记录 | 两库匹配 | 双指标完整 | 匹配/财务 | 匹配/治理 |
|
||||||
|
| --- | --- | --- | --- | --- | --- | --- |
|
||||||
|
| 1990 | 5 | 0 | 0 | 0 | 0.00% | — |
|
||||||
|
| 1991 | 5 | 0 | 0 | 0 | 0.00% | — |
|
||||||
|
| 1992 | 36 | 0 | 0 | 0 | 0.00% | — |
|
||||||
|
| 1993 | 277 | 0 | 0 | 0 | 0.00% | — |
|
||||||
|
| 1994 | 511 | 0 | 0 | 0 | 0.00% | — |
|
||||||
|
| 1995 | 727 | 0 | 0 | 0 | 0.00% | — |
|
||||||
|
| 1996 | 813 | 0 | 0 | 0 | 0.00% | — |
|
||||||
|
| 1997 | 918 | 0 | 0 | 0 | 0.00% | — |
|
||||||
|
| 1998 | 987 | 0 | 0 | 0 | 0.00% | — |
|
||||||
|
| 1999 | 1,088 | 949 | 782 | 776 | 71.88% | 82.40% |
|
||||||
|
| 2000 | 1,161 | 1,108 | 924 | 918 | 79.59% | 83.39% |
|
||||||
|
| 2001 | 1,258 | 1,163 | 977 | 969 | 77.66% | 84.01% |
|
||||||
|
| 2002 | 1,270 | 1,229 | 1,045 | 1,032 | 82.28% | 85.03% |
|
||||||
|
| 2003 | 1,354 | 1,288 | 1,107 | 1,090 | 81.76% | 85.95% |
|
||||||
|
| 2004 | 1,470 | 1,378 | 1,204 | 1,177 | 81.90% | 87.37% |
|
||||||
|
| 2005 | 1,522 | 1,375 | 1,217 | 1,169 | 79.96% | 88.51% |
|
||||||
|
| 2006 | 1,690 | 1,456 | 1,292 | 1,235 | 76.45% | 88.74% |
|
||||||
|
| 2007 | 2,051 | 1,572 | 1,396 | 1,343 | 68.06% | 88.80% |
|
||||||
|
| 2008 | 2,292 | 1,626 | 1,447 | 1,401 | 63.13% | 88.99% |
|
||||||
|
| 2009 | 2,441 | 1,774 | 1,593 | 1,539 | 65.26% | 89.80% |
|
||||||
|
| 2010 | 2,580 | 2,129 | 1,941 | 1,896 | 75.23% | 91.17% |
|
||||||
|
| 2011 | 3,209 | 2,363 | 2,171 | 2,135 | 67.65% | 91.87% |
|
||||||
|
| 2012 | 3,554 | 2,492 | 2,297 | 2,276 | 64.63% | 92.17% |
|
||||||
|
| 2013 | 2,841 | 2,536 | 2,342 | 2,338 | 82.44% | 92.35% |
|
||||||
|
| 2014 | 837 | 2,652 | 836 | 828 | 99.88% | 31.52% |
|
||||||
|
|
||||||
|
“双指标完整”指该年两库匹配且ROA、ROE均非缺失;“—”表示治理文件该年没有记录,分母为零。完整审计表还包括各类未匹配及财务源内缺失数,见 [逐年合并统计](merge_by_year.csv)。
|
||||||
|
|
||||||
|

|
||||||
|
|
||||||
|
图中 RESSET 表示财务文件记录数,CSMAR 表示治理文件记录数,Matched 表示两库匹配数。该图描述文件覆盖,不能解释为市场公司数量或企业退出趋势。
|
||||||
|
|
||||||
|
### 5.2 2014年为何特别需要注意
|
||||||
|
|
||||||
|
2014年财务记录仅837条,治理记录为2,652条,匹配836条。对财务表计算的匹配率为99.88%,但对治理表计算仅为31.52%。较高的前一个比例不能证明2014年财务数据覆盖完整。该年财务文件仅出现000、001、002和600前缀;2013年有2,841条财务记录,2014年的明显减少应首先视为导出覆盖问题,不能直接推断真实经济变化。
|
||||||
|
|
||||||
|
## 六、缺失值、极值与描述统计
|
||||||
|
|
||||||
|
全部财务记录中,ROA缺失69条、ROE缺失726条。匹配后,ROA无缺失,ROE仍缺失449条,故匹配与“双指标完整”是两个不同筛选条件。完整外连接中仅治理记录的ROA/ROE为空属于来源缺失,不应与财务表内部缺失混为一谈。
|
||||||
|
|
||||||
|
以下数值均使用原始尺度,未进行百分数换算。各变量按自身非缺失观测计算;“匹配且双指标完整”使用共同的22,122条记录。
|
||||||
|
|
||||||
|
| 样本 | 变量 | 有效数 | 缺失数 | 均值 | 中位数 | 标准差 | 最小值 | 最大值 |
|
||||||
|
| --- | --- | --- | --- | --- | --- | --- | --- | --- |
|
||||||
|
| 全部财务 | ROA | 34,828 | 69 | -0.00162996 | 0.0498 | 10.471 | -1,950.73 | 64.7546 |
|
||||||
|
| 全部财务 | ROE | 34,171 | 726 | 35.2343 | 0.0958 | 6,512.09 | -1,950.73 | 1.20378e+06 |
|
||||||
|
| 两库匹配 | ROA | 22,571 | 0 | 0.0360606 | 0.0349 | 0.639555 | -64.8192 | 64.7546 |
|
||||||
|
| 两库匹配 | ROE | 22,122 | 449 | 0.019237 | 0.0691 | 3.06854 | -188.392 | 159.903 |
|
||||||
|
| 匹配且双指标完整 | ROA | 22,122 | 0 | 0.0403534 | 0.0355 | 0.106527 | -1.1943 | 7.1963 |
|
||||||
|
| 匹配且双指标完整 | ROE | 22,122 | 0 | 0.019237 | 0.0691 | 3.06854 | -188.392 | 159.903 |
|
||||||
|
|
||||||
|
表中数字为便于阅读而四舍五入;完整精度及P1、P25、P75、P99见 [描述统计文件](descriptive_statistics.csv)。分位点采用Stata `summarize, detail` 的定义,可能与采用线性插值的其他软件不同。
|
||||||
|
|
||||||
|
源文件含明显极端观测,例如证券代码400033在2012年的ROA为−1,950.7321,代码430082在2011年的ROE为1,203,783.875。全部财务ROE均值约35.2343,而中位数为0.0958,说明均值受尾部值显著影响。匹配ROA的中位数为0.0349,匹配ROE的中位数为0.0691;这些只是导出样本的分布描述,不构成总体盈利水平的代表性估计。
|
||||||
|
|
||||||
|
本次没有缩尾、截尾、删除零值或替换异常值,因为题目要求提取与合并原始指标,且没有提供修订依据。极值记录见 [原始极值清单](source_extremes.csv)。后续实证分析若需要异常值处理,应先核验原始财务报表和指标定义,并单独报告处理规则及敏感性结果。
|
||||||
|
|
||||||
|
## 七、用真实数据验证自由度公式
|
||||||
|
|
||||||
|
为将第一题与Stata输出对应,选取2013年两库匹配的横截面,使用ROA作为因变量、董事人数与独立董事比例作为两个解释变量。董事人数须为正,独董比例须在0到1之间。该年2,342条匹配记录全部满足条件,无额外剔除。
|
||||||
|
|
||||||
|
```math
|
||||||
|
ROAᵢ = β₀ + β₁board_sizeᵢ + β₂independent_shareᵢ + uᵢ
|
||||||
|
independent_shareᵢ = 独立董事人数ᵢ / 董事人数ᵢ
|
||||||
|
```
|
||||||
|
|
||||||
|
使用默认普通OLS,仅验证残差自由度与MSE的计算,不将该模型作为治理影响盈利能力的因果识别模型,也不据此断言其误差满足同方差假设。
|
||||||
|
|
||||||
|
```stata
|
||||||
|
regress ROA board_size independent_share if matched & year == 2013 & valid_board
|
||||||
|
display e(N)
|
||||||
|
display e(df_r)
|
||||||
|
display e(rss)/e(df_r)
|
||||||
|
display e(rmse)^2
|
||||||
|
```
|
||||||
|
|
||||||
|
| 项目 | Stata 实际结果 |
|
||||||
|
| --- | --- |
|
||||||
|
| 实际估计样本 n | 2,342 |
|
||||||
|
| 斜率个数 k | 2 |
|
||||||
|
| 包括截距的参数个数 | 3 |
|
||||||
|
| 残差自由度 n − k − 1 | 2,339 |
|
||||||
|
| SSE = e(rss) | 65.691748161464 |
|
||||||
|
| SSE / (n − k − 1) | 0.028085398957 |
|
||||||
|
| e(rmse)² | 0.028085398957 |
|
||||||
|
| SSE / n | 0.028049422785 |
|
||||||
|
|
||||||
|
因此该例恰好满足:
|
||||||
|
|
||||||
|
```math
|
||||||
|
n − k − 1 = 2342 − 2 − 1 = 2339
|
||||||
|
σ̂² = 65.691748161464 / 2339 = 0.028085398957
|
||||||
|
```
|
||||||
|
|
||||||
|
脚本同时用预测残差重新计算SSE,结果与 `e(rss)` 一致。残差之和为3.718e-15,董事人数与残差乘积之和为-3.256e-13,独董比例与残差乘积之和为-1.215e-15,均在浮点计算误差范围内接近零,验证了相应正规方程约束。
|
||||||
|
|
||||||
|
`e(rmse)` 是误差标准差的估计,其平方才是本题中的误差方差估计。回归的完整数值输出见 [Stata运行日志](week8_stata.log),保存的估计结果为 [df_example_ols.ster](df_example_ols.ster),逐项验证数值为 [df_example_metrics.csv](df_example_metrics.csv)。
|
||||||
|
|
||||||
|
## 八、复现步骤与交付文件
|
||||||
|
|
||||||
|
在Stata命令窗口运行:
|
||||||
|
|
||||||
|
```stata
|
||||||
|
do "C:/Users/cattom/Documents/econometrics/Week8/week8_analysis.do"
|
||||||
|
```
|
||||||
|
|
||||||
|
脚本直接读取原始Excel,完成整理、合并、审计、描述统计、图表和OLS示例。无需安装额外Stata命令。重复运行会替换本次生成的同名输出,原始 `data` 目录不被修改。若移动目录,只需修改do-file开头的 `cd` 行。
|
||||||
|
|
||||||
|
也可从PowerShell调用指定Stata程序:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
Start-Process -FilePath 'D:\Stata15\StataSE-64.exe' `
|
||||||
|
-ArgumentList @('/e','do','C:\Users\cattom\Documents\econometrics\Week8\week8_analysis.do') `
|
||||||
|
-WorkingDirectory 'C:\Users\cattom\Documents\econometrics\Week8' `
|
||||||
|
-WindowStyle Hidden -Wait
|
||||||
|
```
|
||||||
|
|
||||||
|
批处理自动产生 `week8_analysis.log`,研究日志为 `week8_stata.log`。成功结束标识为 `WEEK8_ANALYSIS_COMPLETED_SUCCESSFULLY`。
|
||||||
|
|
||||||
|
| 文件 | 用途 |
|
||||||
|
| --- | --- |
|
||||||
|
| [Week8_研究报告.md](Week8_研究报告.md) | 研究过程、推导与结论 |
|
||||||
|
| [Week8_研究报告.html](Week8_研究报告.html) | 可离线打开、带导航和打印样式的HTML5页面 |
|
||||||
|
| [financial_governance_all_years.dta](financial_governance_all_years.dta) | 主交付数据,39,416条,保留全部年份与未匹配记录 |
|
||||||
|
| [financial_governance_all_years.csv](financial_governance_all_years.csv) | 同一主数据的UTF-8通用交换版本 |
|
||||||
|
| [financial_governance_matched.dta](financial_governance_matched.dta) | 两库匹配数据,22,571条,保留ROE缺失记录 |
|
||||||
|
| [resset_financial_all_years.dta](resset_financial_all_years.dta) | 整理后的全部34,897条财务记录 |
|
||||||
|
| [csmar_governance_1999_2014.dta](csmar_governance_1999_2014.dta) | 追加后的27,090条治理记录 |
|
||||||
|
| [week8_analysis.do](week8_analysis.do);[week8_stata.log](week8_stata.log) | 可复现Stata脚本与完整研究运行日志 |
|
||||||
|
| [merge_by_year.csv](merge_by_year.csv);[processing_audit.csv](processing_audit.csv) | 年度覆盖及处理过程计数 |
|
||||||
|
| [unmatched_records.csv](unmatched_records.csv);[source_extremes.csv](source_extremes.csv) | 未匹配记录与原始极值清单 |
|
||||||
|
| [variable_dictionary.csv](variable_dictionary.csv) | 字段、中文含义、单位和来源 |
|
||||||
|
| [descriptive_statistics.csv](descriptive_statistics.csv) | 各样本的有效数、缺失与分布统计 |
|
||||||
|
| [annual_coverage.png](annual_coverage.png);[annual_coverage.gph](annual_coverage.gph) | Stata生成的年度覆盖图与可编辑图形 |
|
||||||
|
| [df_example_ols.ster](df_example_ols.ster);[df_example_sample.dta](df_example_sample.dta) | OLS示例估计结果、样本与残差 |
|
||||||
|
| [df_example_metrics.csv](df_example_metrics.csv) | 残差自由度、SSE、MSE等核验值 |
|
||||||
|
| [validation_results.json](validation_results.json) | 独立复核记录 |
|
||||||
|
|
||||||
|
优先使用DTA以保留证券代码字符串、日期、变量标签和来源标记。CSV导入其他软件时应把 `stkcd` 作为文本读取,避免丢失前导零;CSMAR备注含换行,须用标准CSV解析器读取,不能按文本行数统计观测。
|
||||||
|
|
||||||
|
只需要同时拥有财务与治理资料时,可从主数据 `keep if _merge == 3`;同时要求两个财务指标非缺失时,再加 `keep if complete_roa_roe == 1`。这两个筛选产生22,571和22,122两个不同样本,不应混称。
|
||||||
|
|
||||||
|
Stata 的键唯一性、报告日期一致性及残差自由度断言均已通过。独立核验共170项检查全部通过,其中原始Excel与整理、合并DTA比较了2,485,880个源单元格;数值、缺失值和中文文本均与来源一致,最长3,627字符的备注完整保留。主CSV的55列与DTA逐行一致,12项回归指标在CSV和DTA中也一致。CSV导出时对非日期数值使用完整双精度格式,避免默认显示格式造成大额数值四舍五入。源文件SHA-256及详细检查结果见 `validation_results.json`。
|
||||||
|
|
||||||
|
报告由 `build_report.py` 读取实际Stata汇总结果生成,`build_html.py` 将Markdown渲染成离线HTML;这些辅助脚本不参与替代Stata的数据合并与估计。若数据更换,先重跑Stata,再运行独立核验和两份报告构建脚本。
|
||||||
|
|
||||||
|
## 九、结论与资料依据
|
||||||
|
|
||||||
|
自由度调整的核心是:样本被用于估计参数后,残差可自由变化的维数随之减少。在本题的经典条件下,SSE的期望为 `(n − k − 1)σ²`,所以应除以残差自由度。2013年示例回归的残差自由度为2,339,SSE除以该值与Stata的Root MSE平方一致。
|
||||||
|
|
||||||
|
数据任务已形成保留全部现有年份的可追溯主数据及匹配子集。主要研究限制来自源文件覆盖、ROA/ROE经济尺度尚未获得数据字典确认,以及极值和缺失。现有结果支持描述给定文件的覆盖与数值分布,尚不足以作全市场推断或公司治理的因果结论。
|
||||||
|
|
||||||
|
资料依据包括:用户提供的题目图片与三份原始Excel;原始工作簿字段行和单位行;本机Stata 15官方帮助文档 `D:/Stata15/ado/base/r/regress.sthlp`(Stored results 中对 `e(N)`、`e(df_m)`、`e(rss)`、`e(df_r)`、`e(rmse)` 的定义);本机 `help merge` 对1:1合并及 `_merge` 状态的说明;以及本次实际运行的Stata日志。理论部分由正规方程和投影矩阵性质逐步推导。本次未成功取得在线手册正文,故不引用未经核验的网页页码或数据库指标公式。
|
||||||
|
After Width: | Height: | Size: 153 KiB |
@@ -0,0 +1,103 @@
|
|||||||
|
"""Read-only, independent audit of the three source workbooks."""
|
||||||
|
from pathlib import Path
|
||||||
|
from collections import Counter
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
import math
|
||||||
|
import re
|
||||||
|
import xlrd
|
||||||
|
import pandas as pd
|
||||||
|
|
||||||
|
sys.stdout.reconfigure(encoding="utf-8")
|
||||||
|
ROOT = Path(__file__).resolve().parent
|
||||||
|
DATA = ROOT / "data"
|
||||||
|
out = {}
|
||||||
|
|
||||||
|
f = next(DATA.glob("FIN*.xls"))
|
||||||
|
w = xlrd.open_workbook(f, formatting_info=True)
|
||||||
|
s = w.sheet_by_index(0)
|
||||||
|
rows = [s.row_values(i) for i in range(1, s.nrows)]
|
||||||
|
cols = ["comcd", "name", "stkcd", "b_stkcd", "h_stkcd", "lstflg", "excel_date", "reporttype", "roa", "roe"]
|
||||||
|
fin = pd.DataFrame(rows, columns=cols)
|
||||||
|
fin["date"] = fin.excel_date.map(lambda v: xlrd.xldate_as_datetime(v, w.datemode).strftime("%Y-%m-%d"))
|
||||||
|
fin["year"] = fin.date.str[:4].astype(int)
|
||||||
|
out["finance"] = {
|
||||||
|
"file": f.name, "rows": len(fin), "columns": s.row_values(0),
|
||||||
|
"year_counts": fin.groupby("year").size().to_dict(),
|
||||||
|
"companies": int(fin.stkcd.nunique()),
|
||||||
|
"date_suffixes": dict(Counter(fin.date.str[5:])),
|
||||||
|
"listing_flags": dict(Counter(fin.lstflg)),
|
||||||
|
"report_types": dict(Counter(fin.reporttype)),
|
||||||
|
"missing_a": int((fin.stkcd == "").sum()),
|
||||||
|
"nonsixdigit_code_rows": [
|
||||||
|
{"excel_row":i+2, "stkcd_repr":repr(row["stkcd"]), "cell_type":s.cell(i+1,2).ctype,
|
||||||
|
"comcd":row["comcd"], "name":row["name"], "year":int(row["year"])}
|
||||||
|
for i,row in fin.iterrows() if not re.fullmatch(r"[0-9]{6}",str(row["stkcd"]).strip())
|
||||||
|
],
|
||||||
|
"populated_b": int((fin.b_stkcd != "").sum()),
|
||||||
|
"populated_h": int((fin.h_stkcd != "").sum()),
|
||||||
|
"duplicate_key_rows": int(fin.duplicated(["stkcd", "year"], keep=False).sum()),
|
||||||
|
"duplicate_comcd_year_rows": int(fin.duplicated(["comcd", "year"], keep=False).sum()),
|
||||||
|
"formats": [w.format_map[w.xf_list[s.cell(1,j).xf_index].format_key].format_str for j in range(s.ncols)],
|
||||||
|
}
|
||||||
|
for variable in ("roa", "roe"):
|
||||||
|
original = fin[variable]
|
||||||
|
values = pd.to_numeric(original, errors="coerce")
|
||||||
|
fin[variable] = values
|
||||||
|
out["finance"][variable] = {
|
||||||
|
"missing": int(values.isna().sum()),
|
||||||
|
"unexpected_nonnumeric": original[values.isna() & (original != "")].tolist(),
|
||||||
|
"summary": values.describe(percentiles=[.01, .25, .5, .75, .99]).to_dict(),
|
||||||
|
"min_row": fin.loc[values.idxmin(), ["stkcd", "year", variable]].to_dict(),
|
||||||
|
"max_row": fin.loc[values.idxmax(), ["stkcd", "year", variable]].to_dict(),
|
||||||
|
"annual_summary": fin.groupby("year")[variable].agg(["count", "mean", "median", "min", "max"]).to_dict(orient="index"),
|
||||||
|
}
|
||||||
|
out["finance"]["code_prefix3_counts"] = fin.stkcd.str[:3].value_counts().to_dict()
|
||||||
|
out["finance"]["year_2014_code_prefix_counts"] = fin.loc[fin.year == 2014,"stkcd"].str[:3].value_counts().to_dict()
|
||||||
|
out["finance"]["last_source_rows"] = fin.tail(10)[["comcd", "name", "stkcd", "date", "roa", "roe"]].to_dict(orient="records")
|
||||||
|
|
||||||
|
govs = []
|
||||||
|
out["governance_sources"] = []
|
||||||
|
for f in sorted(DATA.glob("CG*.xls")):
|
||||||
|
w = xlrd.open_workbook(f)
|
||||||
|
s = w.sheet_by_index(0)
|
||||||
|
headers, labels, units = s.row_values(0), s.row_values(1), s.row_values(2)
|
||||||
|
df = pd.DataFrame([s.row_values(i) for i in range(3,s.nrows)], columns=headers)
|
||||||
|
df["year"] = df.Reptdt.str[:4].astype(int)
|
||||||
|
df["source"] = f.name
|
||||||
|
govs.append(df)
|
||||||
|
out["governance_sources"].append({
|
||||||
|
"file": f.name, "rows": len(df), "companies": int(df.Stkcd.nunique()),
|
||||||
|
"year_counts": df.groupby("year").size().to_dict(),
|
||||||
|
"date_suffixes": dict(Counter(df.Reptdt.str[5:])),
|
||||||
|
"duplicates": int(df.duplicated(["Stkcd", "year"], keep=False).sum()),
|
||||||
|
"metadata": [{"variable":a,"label":b,"unit":c} for a,b,c in zip(headers,labels,units)],
|
||||||
|
})
|
||||||
|
gov = pd.concat(govs, ignore_index=True).rename(columns={"Stkcd":"stkcd"})
|
||||||
|
merged = fin.merge(gov, how="outer", on=["stkcd", "year"], validate="1:1", indicator=True)
|
||||||
|
out["merge"] = {
|
||||||
|
"governance_rows": len(gov), "governance_companies": int(gov.stkcd.nunique()),
|
||||||
|
"governance_duplicate_rows": int(gov.duplicated(["stkcd","year"], keep=False).sum()),
|
||||||
|
"counts": {str(k):int(v) for k,v in merged._merge.value_counts().items()},
|
||||||
|
"year_counts": merged.groupby(["year","_merge"], observed=True).size().unstack(fill_value=0).to_dict(orient="index"),
|
||||||
|
"matched_companies": int(merged.loc[merged._merge == "both", "stkcd"].nunique()),
|
||||||
|
"matched_missing_roa": int(merged.loc[merged._merge == "both", "roa"].isna().sum()),
|
||||||
|
"matched_missing_roe": int(merged.loc[merged._merge == "both", "roe"].isna().sum()),
|
||||||
|
"gov_only_codes": sorted(merged.loc[merged._merge == "right_only", "stkcd"].unique()),
|
||||||
|
"gov_only_sample": merged.loc[merged._merge == "right_only", ["stkcd", "year"]].head(20).to_dict(orient="records"),
|
||||||
|
}
|
||||||
|
out["merge"]["matched_complete_both"] = int(((merged._merge == "both") & merged.roa.notna() & merged.roe.notna()).sum())
|
||||||
|
out["merge"]["matched_summary"] = merged.loc[merged._merge == "both", ["roa", "roe"]].describe(percentiles=[.01, .25, .5, .75, .99]).to_dict()
|
||||||
|
out["merge"]["governance_code_prefix3_counts"] = gov.stkcd.str[:3].value_counts().to_dict()
|
||||||
|
def clean_json(value):
|
||||||
|
if isinstance(value, dict):
|
||||||
|
return {str(k): clean_json(v) for k,v in value.items()}
|
||||||
|
if isinstance(value, list):
|
||||||
|
return [clean_json(v) for v in value]
|
||||||
|
if isinstance(value, float) and not math.isfinite(value):
|
||||||
|
return None
|
||||||
|
return value
|
||||||
|
|
||||||
|
out = clean_json(out)
|
||||||
|
(ROOT / "audit_independent.json").write_text(json.dumps(out, ensure_ascii=False, indent=2, default=int, allow_nan=False), encoding="utf-8")
|
||||||
|
print(json.dumps({"finance_rows":len(fin), "governance_rows":len(gov), "merge":out["merge"]["counts"], "audit_path":str(ROOT / "audit_independent.json")}, ensure_ascii=False, indent=2))
|
||||||
@@ -0,0 +1,134 @@
|
|||||||
|
"""Build the offline HTML5 edition of the Week 8 Markdown report."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import html
|
||||||
|
import re
|
||||||
|
from pathlib import Path
|
||||||
|
from urllib.parse import unquote, urlsplit
|
||||||
|
|
||||||
|
import mistune
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parent
|
||||||
|
SOURCE = ROOT / "Week8_研究报告.md"
|
||||||
|
OUTPUT = ROOT / "Week8_研究报告.html"
|
||||||
|
|
||||||
|
|
||||||
|
class ReportRenderer(mistune.HTMLRenderer):
|
||||||
|
def __init__(self):
|
||||||
|
super().__init__(escape=False)
|
||||||
|
self.headings: list[tuple[int, str, str]] = []
|
||||||
|
self.used_ids: set[str] = set()
|
||||||
|
|
||||||
|
def heading(self, text, level, **attrs):
|
||||||
|
label = BeautifulSoup(text, "html.parser").get_text(" ", strip=True)
|
||||||
|
base = re.sub(r"[^\w\u4e00-\u9fff]+", "-", label).strip("-").lower() or "section"
|
||||||
|
anchor, index = base, 2
|
||||||
|
while anchor in self.used_ids:
|
||||||
|
anchor = f"{base}-{index}"
|
||||||
|
index += 1
|
||||||
|
self.used_ids.add(anchor)
|
||||||
|
self.headings.append((level, anchor, label))
|
||||||
|
return f'<h{level} id="{html.escape(anchor, quote=True)}">{text}</h{level}>\n'
|
||||||
|
|
||||||
|
def block_code(self, code, info=None):
|
||||||
|
language = (info or "").strip().split(" ", 1)[0]
|
||||||
|
if language == "math":
|
||||||
|
lines = "\n".join(f'<div>{html.escape(line) or " "}</div>' for line in code.rstrip().splitlines())
|
||||||
|
return f'<div class="equation" role="group" aria-label="数学公式">{lines}</div>\n'
|
||||||
|
label = f'<div class="code-label">{html.escape(language)}</div>' if language else ""
|
||||||
|
return f'<div class="code-block">{label}<pre><code>{html.escape(code)}</code></pre></div>\n'
|
||||||
|
|
||||||
|
def table(self, text):
|
||||||
|
return f'<div class="table-scroll" tabindex="0"><table>{text}</table></div>\n'
|
||||||
|
|
||||||
|
|
||||||
|
CSS = r"""
|
||||||
|
:root{--ink:#183344;--text:#293e4b;--muted:#657986;--teal:#147a78;--line:#dbe5e9;--paper:#fff;--bg:#f0f4f6;--serif:"Cambria Math","STIX Two Math","Times New Roman","Noto Serif SC",serif}
|
||||||
|
*{box-sizing:border-box}html{scroll-behavior:smooth;scroll-padding-top:28px}body{margin:0;background:var(--bg);color:var(--text);font-family:"Segoe UI","Microsoft YaHei","PingFang SC",sans-serif;font-size:16px;line-height:1.85}a{color:#126e76;text-decoration-thickness:1px;text-underline-offset:3px}a:hover{color:#0c4e58}button{font:inherit}a:focus-visible,button:focus-visible,[tabindex]:focus-visible{outline:3px solid #e5a543;outline-offset:4px}.skip-link{position:absolute;left:20px;top:-80px;padding:10px;background:white;z-index:10}.skip-link:focus{top:14px}.masthead{background:#153544;border-top:5px solid #2eaaa0;color:#fff}.masthead-inner{max-width:1260px;margin:auto;padding:42px 42px 38px}.eyebrow{font-size:12px;letter-spacing:.19em;color:#a9d8d7;font-weight:600;margin-bottom:14px}.masthead h1{font-size:clamp(26px,3.2vw,39px);font-weight:650;line-height:1.45;letter-spacing:.01em;max-width:1000px;margin:0}.masthead-bottom{display:flex;align-items:center;justify-content:space-between;gap:20px;margin-top:22px}.masthead-caption{color:#c5d6df;font-size:14px}.header-actions{display:flex;gap:10px;flex-wrap:wrap}.header-actions a,.header-actions button{border:1px solid #6b8c99;background:transparent;color:#edf6f8;border-radius:5px;padding:7px 14px;font-size:13px;text-decoration:none;cursor:pointer;line-height:1.5}.header-actions a:hover,.header-actions button:hover{background:#264d5d}.shell{display:grid;grid-template-columns:230px minmax(0,1fr);gap:34px;max-width:1260px;margin:34px auto 60px;padding:0 34px}.sidebar{min-width:0}.sidebar-inner{position:sticky;top:26px;border-top:2px solid #b1ccd0;padding-top:16px;max-height:calc(100vh - 52px);overflow:auto}.toc-title{font-size:12px;font-weight:700;letter-spacing:.12em;color:#59717d;padding:0 10px 10px}.toc{list-style:none;margin:0;padding:0}.toc li{margin:2px 0}.toc a{display:block;font-size:13px;line-height:1.65;text-decoration:none;padding:7px 10px;color:#4a6674;border-radius:4px;border-left:2px solid transparent}.toc a:hover{background:#e3edef;color:var(--teal);border-left-color:var(--teal)}.toc .level-3 a{padding-left:23px;font-size:12px;color:#6e838e}.sidebar-note{padding:16px 10px 0;margin-top:16px;border-top:1px solid var(--line);color:#738690;font-size:12px}.report{min-width:0;background:var(--paper);padding:36px 42px 42px;border:1px solid #e0e8ec;border-radius:8px;box-shadow:0 4px 18px #17374704}.report>p:first-child{margin-top:0}.report p{margin:15px 0}.report h2{font-size:25px;font-weight:650;line-height:1.5;color:var(--ink);margin:46px 0 18px;padding-top:8px;border-top:1px solid var(--line)}.report h2:first-child{margin-top:0}.report h3{font-size:19px;font-weight:650;line-height:1.65;color:#225466;margin:28px 0 12px}.report h4{font-size:16px;color:var(--ink);margin:22px 0 9px}.report strong{color:#173c4d;font-weight:650}.report ul,.report ol{padding-left:1.55em;margin:14px 0}.report li{padding-left:4px;margin:7px 0}.report li>p{margin:5px 0}.report blockquote{margin:22px 0;padding:13px 19px;border-left:4px solid #2e918e;background:#eef7f6;color:#355c64;border-radius:0 5px 5px 0}.report blockquote p{margin:5px 0}.report hr{border:0;border-top:1px solid var(--line);margin:32px 0}.report code{font-family:"Cascadia Code",Consolas,"Microsoft YaHei",monospace;font-size:.88em;background:#eff4f6;border-radius:3px;padding:2px 5px;overflow-wrap:anywhere}.code-block{margin:20px 0;border:1px solid #dbe5eb;border-radius:6px;overflow:hidden;background:#f5f8fa}.code-label{padding:6px 15px;background:#eaf0f4;border-bottom:1px solid #dbe5eb;font-size:11px;color:#607680;font-weight:650;text-transform:uppercase;letter-spacing:.06em}.report pre{margin:0;padding:16px 18px;line-height:1.65;overflow-x:auto;font-size:13px}.report pre code{padding:0;border:0;border-radius:0;background:transparent;font-size:inherit;overflow-wrap:normal;white-space:pre}.equation{padding:20px 24px;margin:22px 0;background:#fff;border:1px solid #cddfe2;border-left:3px solid #4e9594;border-radius:5px;font-family:var(--serif);font-size:19px;line-height:1.95;text-align:center;overflow-x:auto}.equation>div{min-width:max-content;white-space:pre}.table-scroll{overflow:auto;margin:20px 0 24px;border:1px solid #d9e4e8;border-radius:6px}.table-scroll table{border-collapse:collapse;min-width:100%;font-size:13px;line-height:1.65}.table-scroll th,.table-scroll td{padding:10px 13px;vertical-align:top;border-bottom:1px solid #e4ebee;text-align:left}.table-scroll th{font-weight:650;color:#294c5b;background:#eaf2f4;white-space:nowrap}.table-scroll td{min-width:70px}.table-scroll td code{font-size:12px}.table-scroll tbody tr:nth-child(even){background:#f8fafb}.table-scroll tbody tr:last-child td{border-bottom:0}.table-scroll tbody tr:hover{background:#edf6f5}.report img{display:block;max-width:100%;height:auto;margin:23px auto;border:1px solid #e0e8ec;border-radius:4px}.report a{overflow-wrap:anywhere}.report .footnotes{font-size:13px}.footer{max-width:1260px;margin:0 auto;padding:0 34px 28px;text-align:right;colLine truncated
|
||||||
|
.equation>div{min-width:0;white-space:pre-wrap;overflow-wrap:anywhere}
|
||||||
|
@media(min-width:1450px){.shell{gap:40px}.report{padding:40px 50px 46px}}
|
||||||
|
@media(max-width:950px){.shell{grid-template-columns:185px minmax(0,1fr);gap:20px;padding:0 22px}.report{padding:28px}.masthead-inner{padding:32px 28px}.toc a{font-size:12px}.toc .level-3{display:none}}
|
||||||
|
@media(max-width:720px){body{font-size:15px}.masthead-inner{padding:26px 21px}.masthead-bottom{align-items:flex-start;flex-direction:column;gap:13px}.masthead-caption{font-size:12px}.shell{display:block;margin:20px auto 30px;padding:0 14px}.sidebar{margin:0 2px 20px}.sidebar-inner{position:static;max-height:none;padding-top:12px}.toc-title{padding:0 3px 7px}.toc{display:flex;flex-wrap:wrap;gap:4px}.toc li{margin:0}.toc a{font-size:12px;background:#e5edef;padding:6px 10px}.toc .level-3,.sidebar-note{display:none}.report{padding:24px 19px;border-radius:6px}.report h2{font-size:22px;margin-top:35px}.report h3{font-size:18px}.equation{font-size:17px;padding:14px 16px}.table-scroll th,.table-scroll td{padding:9px 11px}.footer{padding:0 20px 22px;text-align:left}}
|
||||||
|
@media print{@page{size:A4;margin:17mm 16mm}html{scroll-behavior:auto}body{background:white;color:#111;font-size:10.5pt;line-height:1.65}.masthead{background:white!important;color:#122b38;border-top:3px solid #468d88}.masthead-inner{padding:14px 0 19px}.eyebrow{font-size:8pt;color:#486b72;margin-bottom:8px}.masthead h1{font-size:23pt}.masthead-bottom{margin-top:12px}.masthead-caption{font-size:9pt;color:#526b76}.header-actions,.sidebar,.skip-link,.footer{display:none!important}.shell{display:block;margin:0;padding:0;max-width:none}.report{padding:0;border:0;box-shadow:none;border-radius:0}.report h2{font-size:16pt;break-after:avoid;margin-top:24px}.report h3{font-size:12pt;break-after:avoid}.report p,.report li{orphans:3;widows:3}.report a{color:inherit;text-decoration:none}.report pre{white-space:pre-wrap;font-size:8pt;overflow:visible}.report pre code{white-space:pre-wrap;overflow-wrap:anywhere}.code-block{break-inside:avoid}.equation{font-size:12pt;break-inside:avoid;padding:10px 14px}.equation>div{white-space:pre-wrap;min-width:0}.table-scroll{overflow:visible;border-radius:0}.table-scroll table{font-size:8pt;width:100%;table-layout:auto}.table-scroll th,.table-scroll td{padding:6px 7px;min-width:0}.table-scroll th{white-space:normal}thead{display:table-header-group}tr{break-inside:avoid}.report img{max-height:220mm;object-fit:contain;break-inside:avoid}.print-only{display:block}}
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
if not SOURCE.exists():
|
||||||
|
raise SystemExit(f"Markdown report does not exist yet: {SOURCE}")
|
||||||
|
renderer = ReportRenderer()
|
||||||
|
markdown = mistune.create_markdown(renderer=renderer, plugins=["table", "strikethrough", "footnotes"])
|
||||||
|
content = markdown(SOURCE.read_text(encoding="utf-8-sig"))
|
||||||
|
soup = BeautifulSoup(content, "html.parser")
|
||||||
|
first_heading = soup.find("h1")
|
||||||
|
title = first_heading.get_text(" ", strip=True) if first_heading else "第8周计量经济学研究报告"
|
||||||
|
if first_heading:
|
||||||
|
first_heading.decompose()
|
||||||
|
toc = "\n".join(
|
||||||
|
f'<li class="level-{level}"><a href="#{html.escape(anchor, quote=True)}">{html.escape(label)}</a></li>'
|
||||||
|
for level, anchor, label in renderer.headings if level in (2, 3)
|
||||||
|
)
|
||||||
|
actions = f'<a href="{html.escape(SOURCE.name, quote=True)}">Markdown 文档</a>'
|
||||||
|
if (ROOT / "week8_analysis.do").exists():
|
||||||
|
actions += '<a href="week8_analysis.do">Stata 脚本</a>'
|
||||||
|
actions += '<button type="button" onclick="window.print()">打印报告</button>'
|
||||||
|
document = f'''<!doctype html>
|
||||||
|
<html lang="zh-CN">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<meta name="color-scheme" content="light">
|
||||||
|
<meta name="description" content="第8周计量经济学研究报告:回归自由度与财务、治理数据合并。">
|
||||||
|
<title>{html.escape(title)}</title>
|
||||||
|
<style>{CSS}</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<a class="skip-link" href="#report">跳转至正文</a>
|
||||||
|
<header class="masthead">
|
||||||
|
<div class="masthead-inner">
|
||||||
|
<div class="eyebrow">ECONOMETRICS / WEEK 08</div>
|
||||||
|
<h1>{html.escape(title)}</h1>
|
||||||
|
<div class="masthead-bottom">
|
||||||
|
<div class="masthead-caption">计量经济学课程研究报告</div>
|
||||||
|
<div class="header-actions">{actions}</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</header>
|
||||||
|
<div class="shell">
|
||||||
|
<aside class="sidebar" aria-label="报告目录">
|
||||||
|
<div class="sidebar-inner">
|
||||||
|
<div class="toc-title">内容导航</div>
|
||||||
|
<nav aria-label="章节"><ol class="toc">{toc}</ol></nav>
|
||||||
|
<div class="sidebar-note">正文、公式和样式均可离线阅读。关联数据与脚本保留在同一工作目录。</div>
|
||||||
|
</div>
|
||||||
|
</aside>
|
||||||
|
<main class="report" id="report">{soup}</main>
|
||||||
|
</div>
|
||||||
|
<footer class="footer">第8周 · 计量经济学</footer>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
'''
|
||||||
|
OUTPUT.write_text(document, encoding="utf-8")
|
||||||
|
rendered = BeautifulSoup(document, "html.parser")
|
||||||
|
ids = [tag["id"] for tag in rendered.find_all(id=True)]
|
||||||
|
assert len(ids) == len(set(ids)), "Duplicate HTML anchors"
|
||||||
|
for link in rendered.select("nav a[href^='#']"):
|
||||||
|
assert link["href"][1:] in ids, f"Broken navigation: {link['href']}"
|
||||||
|
local_targets = []
|
||||||
|
for element in rendered.select("[href], [src]"):
|
||||||
|
target = element.get("href") or element.get("src")
|
||||||
|
if target.startswith("#") or urlsplit(target).scheme:
|
||||||
|
continue
|
||||||
|
local_targets.append(target)
|
||||||
|
assert (ROOT / unquote(urlsplit(target).path)).exists(), f"Missing local reference: {target}"
|
||||||
|
assert rendered.html.get("lang") == "zh-CN"
|
||||||
|
assert len(rendered.select(".equation")) == len(re.findall(r"^```math\s*$", SOURCE.read_text(encoding="utf-8-sig"), re.M))
|
||||||
|
print(f"Created: {OUTPUT.name}")
|
||||||
|
print(f"Sections: {len(renderer.headings)}; tables: {len(rendered.find_all('table'))}; equations: {len(rendered.select('.equation'))}")
|
||||||
|
print(f"Verified local references: {len(local_targets)}")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,317 @@
|
|||||||
|
"""Write the Chinese report from the actual Stata output tables."""
|
||||||
|
from pathlib import Path
|
||||||
|
import csv
|
||||||
|
import json
|
||||||
|
import math
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parent
|
||||||
|
|
||||||
|
def rows(name):
|
||||||
|
with (ROOT / name).open(encoding="utf-8-sig", newline="") as f:
|
||||||
|
return list(csv.DictReader(f))
|
||||||
|
|
||||||
|
def metrics(name):
|
||||||
|
return {r["metric"]: float(r["value"]) for r in rows(name)}
|
||||||
|
|
||||||
|
def n(x):
|
||||||
|
return f"{int(float(x)):,}"
|
||||||
|
|
||||||
|
def number(x, digits=6):
|
||||||
|
return f"{float(x):,.{digits}g}"
|
||||||
|
|
||||||
|
def table(headers, data):
|
||||||
|
return "\n".join(["| " + " | ".join(headers) + " |", "| " + " | ".join(["---"] * len(headers)) + " |"] + ["| " + " | ".join(map(str, row)) + " |" for row in data])
|
||||||
|
|
||||||
|
audit = metrics("processing_audit.csv")
|
||||||
|
df = metrics("df_example_metrics.csv")
|
||||||
|
yearly = rows("merge_by_year.csv")
|
||||||
|
desc = rows("descriptive_statistics.csv")
|
||||||
|
validation_path = ROOT / "validation_results.json"
|
||||||
|
validation = json.loads(validation_path.read_text(encoding="utf-8"))
|
||||||
|
assert validation["passed"], "Independent source validation did not pass"
|
||||||
|
compared_cells = sum(c.get("cells_compared", 0) for c in validation["checks"] if "source_vs" in c["name"])
|
||||||
|
validation_note = f"Stata 的键唯一性、报告日期一致性及残差自由度断言均已通过。独立核验共{validation['check_count']}项检查全部通过,其中原始Excel与整理、合并DTA比较了{compared_cells:,}个源单元格;数值、缺失值和中文文本均与来源一致,最长3,627字符的备注完整保留。主CSV的55列与DTA逐行一致,12项回归指标在CSV和DTA中也一致。CSV导出时对非日期数值使用完整双精度格式,避免默认显示格式造成大额数值四舍五入。源文件SHA-256及详细检查结果见 `validation_results.json`。"
|
||||||
|
|
||||||
|
year_table = table(
|
||||||
|
["报告年", "财务记录", "治理记录", "两库匹配", "双指标完整", "匹配/财务", "匹配/治理"],
|
||||||
|
[[r["year"], n(r["resset_n"]), n(r["csmar_n"]), n(r["matched_n"]), n(r["complete_n"]), f'{float(r["pct_resset_matched"]):.2f}%', "—" if not r["pct_csmar_matched"] else f'{float(r["pct_csmar_matched"]):.2f}%'] for r in yearly],
|
||||||
|
)
|
||||||
|
samples = {"resset_all": "全部财务", "matched": "两库匹配", "matched_complete": "匹配且双指标完整"}
|
||||||
|
desc_table = table(
|
||||||
|
["样本", "变量", "有效数", "缺失数", "均值", "中位数", "标准差", "最小值", "最大值"],
|
||||||
|
[[samples[r["sample"]], r["variable"], n(r["observations"]), n(r["missing_count"]), *[number(r[k]) for k in ["mean", "p50", "sd", "min", "max"]]] for r in desc],
|
||||||
|
)
|
||||||
|
df_table = table(
|
||||||
|
["项目", "Stata 实际结果"],
|
||||||
|
[["实际估计样本 n", n(df["n_ols"])], ["斜率个数 k", n(df["k_ols"])], ["包括截距的参数个数", n(df["rank_ols"])], ["残差自由度 n − k − 1", n(df["df_ols"])], ["SSE = e(rss)", f'{df["sse_ols"]:.12f}'], ["SSE / (n − k − 1)", f'{df["s2_ols"]:.12f}'], ["e(rmse)²", f'{df["rmse2_ols"]:.12f}'], ["SSE / n", f'{df["naive_s2"]:.12f}']],
|
||||||
|
)
|
||||||
|
matched = audit["merge_status_3"]
|
||||||
|
finance_n = audit["resset_input_rows"]
|
||||||
|
gov_n = audit["csmar_total_rows"]
|
||||||
|
|
||||||
|
report = f'''# 第8周研究报告:回归自由度与财务、治理数据合并
|
||||||
|
|
||||||
|
完成日期:2026年10月29日。软件:本机 Stata/SE 15.1,脚本使用 `version 15.0`。数据截止年份由文件内容确定,为2014年。
|
||||||
|
|
||||||
|
## 一、研究问题与主要结论
|
||||||
|
|
||||||
|
本次工作包括两个问题:解释题目图片中的误差方差估计为何使用分母 `n − k − 1`,以及将所给 RESSET 财务指标 ROA、ROE 与两份 CSMAR 治理综合信息文件整理、合并。[题目原图](1.png)保留在工作目录。
|
||||||
|
|
||||||
|
**理论结论:** 在含截距、设计矩阵满列秩的多元回归中,估计 k 个斜率及一个截距会对残差施加 k + 1 个独立线性约束。因此,残差自由度为 n − k − 1。在条件均值为零、同方差且误差间不相关的条件下,SSE 除以该自由度是误差方差的无偏估计。
|
||||||
|
|
||||||
|
**数据结论:** 财务文件共有 {n(finance_n)} 条记录,覆盖1990—2014年;治理文件追加后共有 {n(gov_n)} 条记录,覆盖1999—2014年。按“证券代码文本+报告年份”一对一外连接,得到 **{n(audit["merged_total_rows"])} 条公司年度记录**,其中 **{n(matched)} 条两库匹配**,匹配且 ROA、ROE 均非缺失的记录为 **{n(audit["matched_complete_roa_roe"])} 条**。
|
||||||
|
|
||||||
|
> 本报告的“所有年份”指所给文件中存在的全部年份。本次直接读取离线导出文件,没有重新登录数据库下载。2014年财务数据覆盖明显不足,文件中还存在非标准代码和非通常沪深A股代码前缀,不能将这些导出记录称为全市场全部上市公司的完整历史。
|
||||||
|
|
||||||
|
## 二、为什么分母要减去 k + 1
|
||||||
|
|
||||||
|
### 2.1 先区分随机误差和估计残差
|
||||||
|
|
||||||
|
设总体回归模型为:
|
||||||
|
|
||||||
|
```math
|
||||||
|
yᵢ = β₀ + β₁xᵢ₁ + ⋯ + βₖxᵢₖ + uᵢ
|
||||||
|
eᵢ = yᵢ − ŷᵢ
|
||||||
|
SSE = ∑ᵢ₌₁ⁿ eᵢ²
|
||||||
|
```
|
||||||
|
|
||||||
|
其中,uᵢ 是不可直接观测的随机误差,eᵢ 是估计模型后得到的残差。n 是实际进入回归的完整观测数,k 是不含截距的解释变量个数。题目中的 σ̂² 估计的是随机误差方差,计算时使用可观测的残差平方和。
|
||||||
|
|
||||||
|
### 2.2 从正规方程理解自由度
|
||||||
|
|
||||||
|
OLS 用同一批观测拟合 k + 1 个参数,使残差平方和尽可能小。一阶条件为:
|
||||||
|
|
||||||
|
```math
|
||||||
|
∑ᵢ₌₁ⁿ eᵢ = 0
|
||||||
|
∑ᵢ₌₁ⁿ xᵢⱼeᵢ = 0,j = 1, …, k
|
||||||
|
```
|
||||||
|
|
||||||
|
当设计矩阵满列秩时,这 k + 1 个条件是独立的线性约束。虽然残差向量有 n 个元素,它只能在 n − k − 1 维空间中变化。这就是残差自由度。这里指线性自由度,并不意味着任取 n − k − 1 个原始残差就一定在概率意义上相互独立。
|
||||||
|
|
||||||
|
只有截距时,估计的是样本均值,分母退化为 n − 1,与样本方差公式相同。简单线性回归估计截距和一个斜率,分母为 n − 2。
|
||||||
|
|
||||||
|
### 2.3 无偏性的矩阵推导
|
||||||
|
|
||||||
|
令 p = k + 1,X 包含截距列,且 rank(X) = p < n。假设条件均值 E(u∣X) = 0,条件方差矩阵 Var(u∣X) = σ²Iₙ。后一个条件同时要求同方差与不同观测误差不相关。
|
||||||
|
|
||||||
|
定义投影矩阵 H 与残差生成矩阵 M:
|
||||||
|
|
||||||
|
```math
|
||||||
|
y = Xβ + u
|
||||||
|
H = X(X′X)⁻¹X′,M = Iₙ − H
|
||||||
|
e = My = Mu
|
||||||
|
M′ = M,M² = M,tr(M) = n − p
|
||||||
|
```
|
||||||
|
|
||||||
|
利用 M 的对称性、幂等性与迹的性质:
|
||||||
|
|
||||||
|
```math
|
||||||
|
SSE = e′e = u′Mu
|
||||||
|
E(SSE∣X) = tr[M E(uu′∣X)]
|
||||||
|
= σ² tr(M)
|
||||||
|
= (n − k − 1)σ²
|
||||||
|
```
|
||||||
|
|
||||||
|
因此:
|
||||||
|
|
||||||
|
```math
|
||||||
|
σ̂² = SSE / (n − k − 1)
|
||||||
|
E(σ̂²∣X) = σ²
|
||||||
|
```
|
||||||
|
|
||||||
|
如果直接除以 n,则 E(SSE/n∣X) = [(n − k − 1)/n]σ²,存在向下偏误。自由度调整补偿了利用样本拟合参数所造成的样本内残差平方和下降。
|
||||||
|
|
||||||
|
例如 n = 100、k = 3、SSE = 192,残差自由度为96,无偏方差估计为192/96 = 2;直接除以100只得到1.92。
|
||||||
|
|
||||||
|
### 2.4 公式的适用边界
|
||||||
|
|
||||||
|
- 一般公式是 `n − rank(X)`。无截距、完全共线性、虚拟变量和交互项等情况,应按实际独立参数数目计算。
|
||||||
|
- 正态性不是上述无偏性的必要条件。正态性用于进一步推导精确的卡方、t 和 F 分布。
|
||||||
|
- 若存在异方差或相关误差,记 Var(u∣X) = Ω,则 E(SSE∣X) = tr(MΩ),通常不能再写成 `(n − k − 1)σ²`。
|
||||||
|
- 自由度调整本身不能解决异方差、内生性或模型遗漏问题。采用企业聚类标准误时,Stata 返回的推断自由度可能以聚类数为基础,不能直接替代本题普通 OLS 的残差自由度。
|
||||||
|
|
||||||
|
## 三、数据来源与变量口径
|
||||||
|
|
||||||
|
| 输入文件 | 工作表 | 数据行数(不含说明行) | 实际报告年份 |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| [RESSET财务文件](<data/FININDRATIO_OLD_5938AD12016_(锐思).xls>) | FININDRATIO_OLD | 34,897 | 1990—2014 |
|
||||||
|
| [CSMAR治理文件1999—2012](data/CG_Ybasic1999-2012.xls) | CG_Ybasic | 21,902 | 1999—2012 |
|
||||||
|
| [CSMAR治理文件2013—2014](data/CG_Ybasic2013-2014.xls) | CG_Ybasic | 5,188 | 2013—2014 |
|
||||||
|
|
||||||
|
RESSET 文件共10列,第1行为中文标题;全部记录的 `Reporttype` 为 Q4,截止日期均为12月31日,`Lstflg` 全为 A,B/H 股票代码列全空。不同证券代码为3,633个。源文件中的“最新公司全称”保留为 `Lcomnm`,不将其视为逐年历史名称。
|
||||||
|
|
||||||
|
CSMAR 两份文件均为33列。第1行为英文字段名,第2行为中文含义,第3行为单位,第4行起才是数据。两份文件追加后包含2,737个不同证券代码。本次保留全部原始治理字段,而非仅保留示例回归使用的董事人数。
|
||||||
|
|
||||||
|
| 分析字段 | 来源字段 | 处理与含义 |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `stkcd` | RESSET `A_Stkcd`;CSMAR `Stkcd` | 作为文本,保留前导零,仅去除首尾空格;通常为六位 |
|
||||||
|
| `year` | RESSET `Enddt`;CSMAR `Reptdt` | 使用报告截止年份,不使用公告年份 |
|
||||||
|
| `ROA` | 资产净利率(%)_ROA | 保留原始数值和缺失值 |
|
||||||
|
| `ROE` | 净资产收益率(摊薄)(%)_ROE | 保留摊薄口径的原始数值和缺失值 |
|
||||||
|
| `Y1101a` | CSMAR 董事人数 | 原始数值字段,示例中构造 `board_size` |
|
||||||
|
| `Y1101b` | CSMAR 独立董事人数 | 与董事人数之比构成 `independent_share` |
|
||||||
|
| `_merge` | Stata `merge` 生成 | 1:仅财务;2:仅治理;3:两库匹配 |
|
||||||
|
| `complete_roa_roe` | 派生标记 | 两库匹配且ROA、ROE均非缺失时为1 |
|
||||||
|
| `code_format_exception` | 派生标记 | 原始证券代码非六位时为1 |
|
||||||
|
|
||||||
|
完整字段含义和单位见 [变量字典](variable_dictionary.csv)。`resset_excel_row`、`csmar_source` 和 `csmar_excel_row` 可追溯原始工作簿与行号。
|
||||||
|
|
||||||
|
**ROA、ROE 单位处理:** 原列标题写有“(%)”,但 Excel 单元格格式为 General,未提供指标公式或完整数据库数据字典。仅凭标题、格式和数值分布,不能独立确认导出数值究竟采用百分数尺度还是比例小数尺度。本次不乘100、不除100,所有统计均以“原始数值尺度”报告,不将0.0498擅自解释为0.0498%或4.98%。
|
||||||
|
|
||||||
|
## 四、Stata 数据整理与合并方法
|
||||||
|
|
||||||
|
1. 从RESSET工作表第2行读取数值,保存所有既有年份。验证Q4、12月31日及公司年度键唯一性,保留ROA、ROE原值。
|
||||||
|
2. 分别导入两份CSMAR表,读取中文含义与单位后去掉两行说明;将人数、持股量、薪酬等字段转换为数值。未使用 `destring, force`,防止无提示地把异常文本变成缺失。
|
||||||
|
3. 使用 `append` 将1999—2012和2013—2014治理数据纵向拼接,验证 `isid stkcd year`。
|
||||||
|
4. 以财务数据为master、治理数据为using,执行 `merge 1:1 stkcd year`。保留三种合并状态,并核验匹配记录的完整截止日期相等。
|
||||||
|
5. 保存完整外连接数据集,另存匹配子集;在汇总表中分别报告源数据缺失与对方来源未出现,不用零替代缺失。
|
||||||
|
|
||||||
|
核心合并代码与完整脚本中的实现一致:
|
||||||
|
|
||||||
|
```stata
|
||||||
|
use "resset_financial_all_years.dta", clear
|
||||||
|
isid stkcd year
|
||||||
|
merge 1:1 stkcd year using "csmar_governance_1999_2014.dta"
|
||||||
|
assert Enddt == csmar_date if _merge == 3
|
||||||
|
isid stkcd year
|
||||||
|
```
|
||||||
|
|
||||||
|
这里 `append` 增加治理表的年份记录,`merge` 按同一公司、同一报告年份横向增加变量。未使用 `merge m:m`,也未通过强制删除重复键来让合并继续;两端原始公司年度键本身均唯一。
|
||||||
|
|
||||||
|
**代码例外:** RESSET 的Excel第14580、14581行,吉林通海高科技股份有限公司1998、1999年的股票代码原值均为文本 `0991`。CSMAR 中既没有 `0991`,也没有 `000991`。本次保留 `0991` 并标记,未推测补成另一证券代码,两条均保留在仅财务记录中。
|
||||||
|
|
||||||
|
## 五、合并结果与年度覆盖
|
||||||
|
|
||||||
|
| 状态 | 条数 | 解释 |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `_merge = 1` | {n(audit["merge_status_1"])} | 财务表存在,对应公司年度不在治理表 |
|
||||||
|
| `_merge = 2` | {n(audit["merge_status_2"])} | 治理表存在,对应公司年度不在财务表 |
|
||||||
|
| `_merge = 3` | {n(matched)} | 同一证券代码、同一报告年份两库均存在 |
|
||||||
|
| 完整外连接 | {n(audit["merged_total_rows"])} | 三类记录合计 |
|
||||||
|
| 匹配且双指标完整 | {n(audit["matched_complete_roa_roe"])} | 是匹配记录的子集,不与前三类重复相加 |
|
||||||
|
|
||||||
|
核对关系为:`34,897 + 27,090 − 22,571 = 39,416`。匹配率以财务记录为分母为 **{matched/finance_n:.2%}**,以治理记录为分母为 **{matched/gov_n:.2%}**。不同分母回答不同覆盖问题,不应混用。
|
||||||
|
|
||||||
|
仅财务的12,326条中,4,279条来自1990—1998年,早于治理文件起始年份;其余8,047条处于1999—2014年。仅治理的4,519条说明对应财务键未在所给文件中出现,不能据此判断公司当年没有财务报告。
|
||||||
|
|
||||||
|
两库匹配子集覆盖1999—2014年,共2,376个不同证券代码。该集合是非平衡公司年度面板,不能假定每家公司在每年都有完整记录。原始文件未提供足够的证券类别、历史上市状态及跨代码公司映射资料,本次未按公司名称或代码前缀强行归并实体。
|
||||||
|
|
||||||
|
### 5.1 全部年份的覆盖表
|
||||||
|
|
||||||
|
{year_table}
|
||||||
|
|
||||||
|
“双指标完整”指该年两库匹配且ROA、ROE均非缺失;“—”表示治理文件该年没有记录,分母为零。完整审计表还包括各类未匹配及财务源内缺失数,见 [逐年合并统计](merge_by_year.csv)。
|
||||||
|
|
||||||
|

|
||||||
|
|
||||||
|
图中 RESSET 表示财务文件记录数,CSMAR 表示治理文件记录数,Matched 表示两库匹配数。该图描述文件覆盖,不能解释为市场公司数量或企业退出趋势。
|
||||||
|
|
||||||
|
### 5.2 2014年为何特别需要注意
|
||||||
|
|
||||||
|
2014年财务记录仅837条,治理记录为2,652条,匹配836条。对财务表计算的匹配率为99.88%,但对治理表计算仅为31.52%。较高的前一个比例不能证明2014年财务数据覆盖完整。该年财务文件仅出现000、001、002和600前缀;2013年有2,841条财务记录,2014年的明显减少应首先视为导出覆盖问题,不能直接推断真实经济变化。
|
||||||
|
|
||||||
|
## 六、缺失值、极值与描述统计
|
||||||
|
|
||||||
|
全部财务记录中,ROA缺失69条、ROE缺失726条。匹配后,ROA无缺失,ROE仍缺失449条,故匹配与“双指标完整”是两个不同筛选条件。完整外连接中仅治理记录的ROA/ROE为空属于来源缺失,不应与财务表内部缺失混为一谈。
|
||||||
|
|
||||||
|
以下数值均使用原始尺度,未进行百分数换算。各变量按自身非缺失观测计算;“匹配且双指标完整”使用共同的22,122条记录。
|
||||||
|
|
||||||
|
{desc_table}
|
||||||
|
|
||||||
|
表中数字为便于阅读而四舍五入;完整精度及P1、P25、P75、P99见 [描述统计文件](descriptive_statistics.csv)。分位点采用Stata `summarize, detail` 的定义,可能与采用线性插值的其他软件不同。
|
||||||
|
|
||||||
|
源文件含明显极端观测,例如证券代码400033在2012年的ROA为−1,950.7321,代码430082在2011年的ROE为1,203,783.875。全部财务ROE均值约35.2343,而中位数为0.0958,说明均值受尾部值显著影响。匹配ROA的中位数为0.0349,匹配ROE的中位数为0.0691;这些只是导出样本的分布描述,不构成总体盈利水平的代表性估计。
|
||||||
|
|
||||||
|
本次没有缩尾、截尾、删除零值或替换异常值,因为题目要求提取与合并原始指标,且没有提供修订依据。极值记录见 [原始极值清单](source_extremes.csv)。后续实证分析若需要异常值处理,应先核验原始财务报表和指标定义,并单独报告处理规则及敏感性结果。
|
||||||
|
|
||||||
|
## 七、用真实数据验证自由度公式
|
||||||
|
|
||||||
|
为将第一题与Stata输出对应,选取2013年两库匹配的横截面,使用ROA作为因变量、董事人数与独立董事比例作为两个解释变量。董事人数须为正,独董比例须在0到1之间。该年2,342条匹配记录全部满足条件,无额外剔除。
|
||||||
|
|
||||||
|
```math
|
||||||
|
ROAᵢ = β₀ + β₁board_sizeᵢ + β₂independent_shareᵢ + uᵢ
|
||||||
|
independent_shareᵢ = 独立董事人数ᵢ / 董事人数ᵢ
|
||||||
|
```
|
||||||
|
|
||||||
|
使用默认普通OLS,仅验证残差自由度与MSE的计算,不将该模型作为治理影响盈利能力的因果识别模型,也不据此断言其误差满足同方差假设。
|
||||||
|
|
||||||
|
```stata
|
||||||
|
regress ROA board_size independent_share if matched & year == 2013 & valid_board
|
||||||
|
display e(N)
|
||||||
|
display e(df_r)
|
||||||
|
display e(rss)/e(df_r)
|
||||||
|
display e(rmse)^2
|
||||||
|
```
|
||||||
|
|
||||||
|
{df_table}
|
||||||
|
|
||||||
|
因此该例恰好满足:
|
||||||
|
|
||||||
|
```math
|
||||||
|
n − k − 1 = 2342 − 2 − 1 = 2339
|
||||||
|
σ̂² = 65.691748161464 / 2339 = 0.028085398957
|
||||||
|
```
|
||||||
|
|
||||||
|
脚本同时用预测残差重新计算SSE,结果与 `e(rss)` 一致。残差之和为{df["residual_sum"]:.3e},董事人数与残差乘积之和为{df["board_residual_sum"]:.3e},独董比例与残差乘积之和为{df["independent_residual_sum"]:.3e},均在浮点计算误差范围内接近零,验证了相应正规方程约束。
|
||||||
|
|
||||||
|
`e(rmse)` 是误差标准差的估计,其平方才是本题中的误差方差估计。回归的完整数值输出见 [Stata运行日志](week8_stata.log),保存的估计结果为 [df_example_ols.ster](df_example_ols.ster),逐项验证数值为 [df_example_metrics.csv](df_example_metrics.csv)。
|
||||||
|
|
||||||
|
## 八、复现步骤与交付文件
|
||||||
|
|
||||||
|
在Stata命令窗口运行:
|
||||||
|
|
||||||
|
```stata
|
||||||
|
do "C:/Users/cattom/Documents/econometrics/Week8/week8_analysis.do"
|
||||||
|
```
|
||||||
|
|
||||||
|
脚本直接读取原始Excel,完成整理、合并、审计、描述统计、图表和OLS示例。无需安装额外Stata命令。重复运行会替换本次生成的同名输出,原始 `data` 目录不被修改。若移动目录,只需修改do-file开头的 `cd` 行。
|
||||||
|
|
||||||
|
也可从PowerShell调用指定Stata程序:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
Start-Process -FilePath 'D:\\Stata15\\StataSE-64.exe' `
|
||||||
|
-ArgumentList @('/e','do','C:\\Users\\cattom\\Documents\\econometrics\\Week8\\week8_analysis.do') `
|
||||||
|
-WorkingDirectory 'C:\\Users\\cattom\\Documents\\econometrics\\Week8' `
|
||||||
|
-WindowStyle Hidden -Wait
|
||||||
|
```
|
||||||
|
|
||||||
|
批处理自动产生 `week8_analysis.log`,研究日志为 `week8_stata.log`。成功结束标识为 `WEEK8_ANALYSIS_COMPLETED_SUCCESSFULLY`。
|
||||||
|
|
||||||
|
| 文件 | 用途 |
|
||||||
|
| --- | --- |
|
||||||
|
| [Week8_研究报告.md](Week8_研究报告.md) | 研究过程、推导与结论 |
|
||||||
|
| [Week8_研究报告.html](Week8_研究报告.html) | 可离线打开、带导航和打印样式的HTML5页面 |
|
||||||
|
| [financial_governance_all_years.dta](financial_governance_all_years.dta) | 主交付数据,39,416条,保留全部年份与未匹配记录 |
|
||||||
|
| [financial_governance_all_years.csv](financial_governance_all_years.csv) | 同一主数据的UTF-8通用交换版本 |
|
||||||
|
| [financial_governance_matched.dta](financial_governance_matched.dta) | 两库匹配数据,22,571条,保留ROE缺失记录 |
|
||||||
|
| [resset_financial_all_years.dta](resset_financial_all_years.dta) | 整理后的全部34,897条财务记录 |
|
||||||
|
| [csmar_governance_1999_2014.dta](csmar_governance_1999_2014.dta) | 追加后的27,090条治理记录 |
|
||||||
|
| [week8_analysis.do](week8_analysis.do);[week8_stata.log](week8_stata.log) | 可复现Stata脚本与完整研究运行日志 |
|
||||||
|
| [merge_by_year.csv](merge_by_year.csv);[processing_audit.csv](processing_audit.csv) | 年度覆盖及处理过程计数 |
|
||||||
|
| [unmatched_records.csv](unmatched_records.csv);[source_extremes.csv](source_extremes.csv) | 未匹配记录与原始极值清单 |
|
||||||
|
| [variable_dictionary.csv](variable_dictionary.csv) | 字段、中文含义、单位和来源 |
|
||||||
|
| [descriptive_statistics.csv](descriptive_statistics.csv) | 各样本的有效数、缺失与分布统计 |
|
||||||
|
| [annual_coverage.png](annual_coverage.png);[annual_coverage.gph](annual_coverage.gph) | Stata生成的年度覆盖图与可编辑图形 |
|
||||||
|
| [df_example_ols.ster](df_example_ols.ster);[df_example_sample.dta](df_example_sample.dta) | OLS示例估计结果、样本与残差 |
|
||||||
|
| [df_example_metrics.csv](df_example_metrics.csv) | 残差自由度、SSE、MSE等核验值 |
|
||||||
|
| [validation_results.json](validation_results.json) | 独立复核记录 |
|
||||||
|
|
||||||
|
优先使用DTA以保留证券代码字符串、日期、变量标签和来源标记。CSV导入其他软件时应把 `stkcd` 作为文本读取,避免丢失前导零;CSMAR备注含换行,须用标准CSV解析器读取,不能按文本行数统计观测。
|
||||||
|
|
||||||
|
只需要同时拥有财务与治理资料时,可从主数据 `keep if _merge == 3`;同时要求两个财务指标非缺失时,再加 `keep if complete_roa_roe == 1`。这两个筛选产生22,571和22,122两个不同样本,不应混称。
|
||||||
|
|
||||||
|
{validation_note}
|
||||||
|
|
||||||
|
报告由 `build_report.py` 读取实际Stata汇总结果生成,`build_html.py` 将Markdown渲染成离线HTML;这些辅助脚本不参与替代Stata的数据合并与估计。若数据更换,先重跑Stata,再运行独立核验和两份报告构建脚本。
|
||||||
|
|
||||||
|
## 九、结论与资料依据
|
||||||
|
|
||||||
|
自由度调整的核心是:样本被用于估计参数后,残差可自由变化的维数随之减少。在本题的经典条件下,SSE的期望为 `(n − k − 1)σ²`,所以应除以残差自由度。2013年示例回归的残差自由度为2,339,SSE除以该值与Stata的Root MSE平方一致。
|
||||||
|
|
||||||
|
数据任务已形成保留全部现有年份的可追溯主数据及匹配子集。主要研究限制来自源文件覆盖、ROA/ROE经济尺度尚未获得数据字典确认,以及极值和缺失。现有结果支持描述给定文件的覆盖与数值分布,尚不足以作全市场推断或公司治理的因果结论。
|
||||||
|
|
||||||
|
资料依据包括:用户提供的题目图片与三份原始Excel;原始工作簿字段行和单位行;本机Stata 15官方帮助文档 `D:/Stata15/ado/base/r/regress.sthlp`(Stored results 中对 `e(N)`、`e(df_m)`、`e(rss)`、`e(df_r)`、`e(rmse)` 的定义);本机 `help merge` 对1:1合并及 `_merge` 状态的说明;以及本次实际运行的Stata日志。理论部分由正规方程和投影矩阵性质逐步推导。本次未成功取得在线手册正文,故不引用未经核验的网页页码或数据库指标公式。
|
||||||
|
'''
|
||||||
|
|
||||||
|
(ROOT / "Week8_研究报告.md").write_text(report, encoding="utf-8")
|
||||||
|
print("Created Week8_研究报告.md")
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
sample,variable,observations,missing_count,mean,sd,min,p1,p25,p50,p75,p99,max
|
||||||
|
resset_all,ROA,34828,69,-.0016299615252095999,10.47100804707682,-1950.7320999999999,-.2994,.017899999999999999,.049799999999999997,.093450000000000005,.35410000000000003,64.754599999999996
|
||||||
|
resset_all,ROE,34171,726,35.234277950542861,6512.0885619172932,-1950.7320560000001,-.86380000000000001,.042599999999999999,.095799999999999996,.17330000000000001,.58169999999999999,1203783.875
|
||||||
|
matched,ROA,22571,0,.036060564441096982,.63955469537421616,-64.819199999999995,-.26240000000000002,.0115,.0349,.066500000000000004,.2341,64.754599999999996
|
||||||
|
matched,ROE,22122,449,.019236986167615946,3.0685399341260502,-188.392303,-1.0933999999999999,.028299999999999999,.069099999999999995,.1125,.37569999999999998,159.90280200000001
|
||||||
|
matched_complete,ROA,22122,0,.040353408371756619,.10652695305733652,-1.1942999999999999,-.18690000000000001,.012200000000000001,.035499999999999997,.066799999999999998,.222,7.1962999999999999
|
||||||
|
matched_complete,ROE,22122,0,.019236986167615946,3.0685399341260502,-188.392303,-1.0933999999999999,.028299999999999999,.069099999999999995,.1125,.37569999999999998,159.90280200000001
|
||||||
|
@@ -0,0 +1,13 @@
|
|||||||
|
metric,value
|
||||||
|
"n_ols",2342
|
||||||
|
"k_ols",2
|
||||||
|
"rank_ols",3
|
||||||
|
"df_ols",2339
|
||||||
|
"sse_ols",65.691748161464361
|
||||||
|
"s2_ols",.028085398957445218
|
||||||
|
"rmse2_ols",.028085398957445214
|
||||||
|
"naive_s2",.028049422784570607
|
||||||
|
"residual_sse",65.691748161464361
|
||||||
|
"residual_sum",3.7175124090182976e-15
|
||||||
|
"board_residual_sum",-3.2559024920608692e-13
|
||||||
|
"independent_residual_sum",-1.2151636305263863e-15
|
||||||
|
@@ -0,0 +1,18 @@
|
|||||||
|
version 15.0
|
||||||
|
clear all
|
||||||
|
set more off
|
||||||
|
set linesize 220
|
||||||
|
capture log close _all
|
||||||
|
cd "C:/Users/cattom/Documents/econometrics/Week8"
|
||||||
|
log using "input_inspection_details.log", text replace
|
||||||
|
foreach f in "FININDRATIO_OLD_5938AD12016_(锐思).xls" "CG_Ybasic1999-2012.xls" "CG_Ybasic2013-2014.xls" {
|
||||||
|
display "===== FILE: `f' ====="
|
||||||
|
import excel using "data/`f'", describe
|
||||||
|
import excel using "data/`f'", clear allstring
|
||||||
|
describe
|
||||||
|
list in 1/6, noobs clean
|
||||||
|
count
|
||||||
|
}
|
||||||
|
display "INSPECTION_COMPLETED"
|
||||||
|
log close
|
||||||
|
exit, clear
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
year,resset_n,csmar_n,matched_n,complete_n,resset_only,csmar_only,missing_roa,missing_roe,pct_resset_matched,pct_csmar_matched
|
||||||
|
1990,5,0,0,0,5,0,0,1,0.000000,
|
||||||
|
1991,5,0,0,0,5,0,0,5,0.000000,
|
||||||
|
1992,36,0,0,0,36,0,0,22,0.000000,
|
||||||
|
1993,277,0,0,0,277,0,3,0,0.000000,
|
||||||
|
1994,511,0,0,0,511,0,1,0,0.000000,
|
||||||
|
1995,727,0,0,0,727,0,0,0,0.000000,
|
||||||
|
1996,813,0,0,0,813,0,1,0,0.000000,
|
||||||
|
1997,918,0,0,0,918,0,1,0,0.000000,
|
||||||
|
1998,987,0,0,0,987,0,1,2,0.000000,
|
||||||
|
1999,1088,949,782,776,306,167,5,9,71.875000,82.402529
|
||||||
|
2000,1161,1108,924,918,237,184,0,10,79.586563,83.393502
|
||||||
|
2001,1258,1163,977,969,281,186,0,17,77.662957,84.006879
|
||||||
|
2002,1270,1229,1045,1032,225,184,0,27,82.283465,85.028478
|
||||||
|
2003,1354,1288,1107,1090,247,181,0,37,81.757755,85.947205
|
||||||
|
2004,1470,1378,1204,1177,266,174,8,53,81.904762,87.373004
|
||||||
|
2005,1522,1375,1217,1169,305,158,6,74,79.960578,88.509091
|
||||||
|
2006,1690,1456,1292,1235,398,164,3,82,76.449704,88.736264
|
||||||
|
2007,2051,1572,1396,1343,655,176,6,75,68.064359,88.804071
|
||||||
|
2008,2292,1626,1447,1401,845,179,5,65,63.132635,88.991390
|
||||||
|
2009,2441,1774,1593,1539,848,181,9,70,65.260139,89.797069
|
||||||
|
2010,2580,2129,1941,1896,639,188,7,64,75.232558,91.169563
|
||||||
|
2011,3209,2363,2171,2135,1038,192,2,59,67.653475,91.874736
|
||||||
|
2012,3554,2492,2297,2276,1257,195,9,41,64.631401,92.174960
|
||||||
|
2013,2841,2536,2342,2338,499,194,2,5,82.435762,92.350158
|
||||||
|
2014,837,2652,836,828,1,1816,0,8,99.880526,31.523379
|
||||||
|
@@ -0,0 +1,25 @@
|
|||||||
|
metric,value
|
||||||
|
"resset_input_rows",34897
|
||||||
|
"resset_code_format_exceptions",2
|
||||||
|
"resset_duplicate_key_excess",0
|
||||||
|
"resset_min_year",1990
|
||||||
|
"resset_max_year",2014
|
||||||
|
"resset_distinct_codes",3633
|
||||||
|
"resset_missing_ROA",69
|
||||||
|
"resset_missing_ROE",726
|
||||||
|
"csmar_1999-2012_input_rows",21902
|
||||||
|
"csmar_2013-2014_input_rows",5188
|
||||||
|
"csmar_total_rows",27090
|
||||||
|
"csmar_duplicate_key_excess",0
|
||||||
|
"csmar_distinct_codes",2737
|
||||||
|
"merged_total_rows",39416
|
||||||
|
"merge_status_1",12326
|
||||||
|
"merge_status_2",4519
|
||||||
|
"merge_status_3",22571
|
||||||
|
"matched_complete_roa_roe",22122
|
||||||
|
"matched_distinct_codes",2376
|
||||||
|
"resset_only_before_1999",4279
|
||||||
|
"resset_only_from_1999",8047
|
||||||
|
"matched_missing_ROA",0
|
||||||
|
"matched_missing_ROE",449
|
||||||
|
"ols_2013_excluded_board_records",0
|
||||||
|
|
After Width: | Height: | Size: 73 KiB |
|
After Width: | Height: | Size: 1.0 MiB |
|
After Width: | Height: | Size: 84 KiB |
|
After Width: | Height: | Size: 1.3 MiB |
|
After Width: | Height: | Size: 152 KiB |
@@ -0,0 +1,4 @@
|
|||||||
|
stkcd,year,ROA,ROE,Comcd,Lcomnm,resset_excel_row
|
||||||
|
"000506",2007,64.754599999999996,,"C000506","中润资源投资股份有限公司",1633
|
||||||
|
"400033",2012,-1950.7320999999999,-1950.7320560000001,"C400033","长春高斯达生物科技集团股份有限公司",17445
|
||||||
|
"430082",2011,.099400000000000002,1203783.875,"C430082","北京博雅英杰科技股份有限公司",18313
|
||||||
|
@@ -0,0 +1,134 @@
|
|||||||
|
"""Independently reconcile Stata datasets to every supplied Excel source cell."""
|
||||||
|
from pathlib import Path
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
import math
|
||||||
|
import hashlib
|
||||||
|
import xlrd
|
||||||
|
import numpy as np
|
||||||
|
import pandas as pd
|
||||||
|
|
||||||
|
sys.stdout.reconfigure(encoding="utf-8")
|
||||||
|
ROOT = Path(__file__).resolve().parent
|
||||||
|
results = {"method":"xlrd reads original source cells; pandas independently reads Stata dta", "checks":[], "passed":True}
|
||||||
|
|
||||||
|
def check(name, passed, **details):
|
||||||
|
results["checks"].append({"name":name,"passed":bool(passed),**details})
|
||||||
|
if not passed:
|
||||||
|
results["passed"] = False
|
||||||
|
print("FAIL", name, details)
|
||||||
|
|
||||||
|
def compare_frames(name, expected, actual, require_exact_numeric=False):
|
||||||
|
check(name+" row_count",len(expected)==len(actual), expected=len(expected), actual=len(actual))
|
||||||
|
check(name+" key_alignment",expected.index.equals(actual.index))
|
||||||
|
if not expected.index.equals(actual.index):
|
||||||
|
actual = actual.reindex(expected.index)
|
||||||
|
for col in expected.columns:
|
||||||
|
if col not in actual:
|
||||||
|
check(name+" column "+col,False,reason="missing column")
|
||||||
|
continue
|
||||||
|
exp, got = expected[col], actual[col]
|
||||||
|
if pd.api.types.is_datetime64_any_dtype(exp.dtype):
|
||||||
|
ok = ((exp == got) | (exp.isna() & got.isna()))
|
||||||
|
detail = {"kind":"date"}
|
||||||
|
elif pd.api.types.is_numeric_dtype(got.dtype):
|
||||||
|
exp = pd.to_numeric(exp.mask(exp.eq(""), np.nan),errors="raise").astype(float)
|
||||||
|
got = got.astype(float)
|
||||||
|
exact = (exp == got) | (exp.isna() & got.isna())
|
||||||
|
ok = exact if require_exact_numeric else np.isclose(exp.to_numpy(),got.to_numpy(),rtol=1e-13,atol=1e-13,equal_nan=True)
|
||||||
|
delta = np.abs(exp-got)
|
||||||
|
detail = {"kind":"numeric","exact_equal_cells":int(exact.sum()),"missing_source":int(exp.isna().sum()),"missing_output":int(got.isna().sum()),"max_absolute_difference":float(delta.max()) if delta.notna().any() else 0}
|
||||||
|
else:
|
||||||
|
exp = exp.fillna("").astype(str)
|
||||||
|
got = got.fillna("").astype(str)
|
||||||
|
ok = exp == got
|
||||||
|
detail = {"kind":"text","max_source_characters":int(exp.str.len().max()),"max_output_characters":int(got.str.len().max())}
|
||||||
|
ok = np.asarray(ok)
|
||||||
|
detail["cells_compared"] = int(len(exp))
|
||||||
|
detail["mismatches"] = int((~ok).sum())
|
||||||
|
if not ok.all():
|
||||||
|
bad = np.flatnonzero(~ok)[:5]
|
||||||
|
detail["examples"] = [{"key":str(exp.index[i]),"source":str(exp.iloc[i]),"output":str(got.iloc[i])} for i in bad]
|
||||||
|
check(name+" column "+col,ok.all(),**detail)
|
||||||
|
|
||||||
|
financial_file = next((ROOT/"data").glob("FIN*.xls"))
|
||||||
|
w=xlrd.open_workbook(financial_file)
|
||||||
|
s=w.sheet_by_index(0)
|
||||||
|
financial_cols=["Comcd","Lcomnm","A_Stkcd","B_Stkcd","H_Stkcd","Lstflg","Enddt","Reporttype","ROA","ROE"]
|
||||||
|
financial=pd.DataFrame([s.row_values(i) for i in range(1,s.nrows)],columns=financial_cols)
|
||||||
|
financial["Enddt"]=pd.to_datetime(financial.Enddt.map(lambda x:xlrd.xldate_as_datetime(x,w.datemode)))
|
||||||
|
financial["resset_excel_row"]=np.arange(2,s.nrows+1)
|
||||||
|
financial=financial.set_index("resset_excel_row").sort_index()
|
||||||
|
fin_dta=pd.read_stata(ROOT/"resset_financial_all_years.dta",convert_categoricals=False)
|
||||||
|
compare_frames("financial_source_vs_clean",financial,fin_dta.set_index("resset_excel_row").sort_index())
|
||||||
|
|
||||||
|
govs=[]
|
||||||
|
source_hashes={financial_file.name:hashlib.sha256(financial_file.read_bytes()).hexdigest()}
|
||||||
|
for f in sorted((ROOT/"data").glob("CG*.xls")):
|
||||||
|
w=xlrd.open_workbook(f)
|
||||||
|
s=w.sheet_by_index(0)
|
||||||
|
df=pd.DataFrame([s.row_values(i) for i in range(3,s.nrows)],columns=s.row_values(0))
|
||||||
|
df=df.rename(columns={"Stkcd":"stkcd"})
|
||||||
|
df["csmar_source"]=f.name
|
||||||
|
df["csmar_excel_row"]=np.arange(4,s.nrows+1)
|
||||||
|
govs.append(df)
|
||||||
|
source_hashes[f.name]=hashlib.sha256(f.read_bytes()).hexdigest()
|
||||||
|
governance=pd.concat(govs,ignore_index=True).set_index(["csmar_source","csmar_excel_row"]).sort_index()
|
||||||
|
gov_dta=pd.read_stata(ROOT/"csmar_governance_1999_2014.dta",convert_categoricals=False)
|
||||||
|
compare_frames("governance_source_vs_clean",governance,gov_dta.set_index(["csmar_source","csmar_excel_row"]).sort_index())
|
||||||
|
|
||||||
|
merged=pd.read_stata(ROOT/"financial_governance_all_years.dta",convert_categoricals=False)
|
||||||
|
check("merged_key_unique",not merged.duplicated(["stkcd","year"]).any())
|
||||||
|
compare_frames("financial_source_vs_merged",financial,merged.loc[merged.in_resset==1].set_index("resset_excel_row").sort_index())
|
||||||
|
compare_frames("governance_source_vs_merged",governance,merged.loc[merged.in_csmar==1].set_index(["csmar_source","csmar_excel_row"]).sort_index())
|
||||||
|
|
||||||
|
expected_fin=financial.reset_index()[["A_Stkcd","Enddt"]].rename(columns={"A_Stkcd":"stkcd"})
|
||||||
|
expected_fin["year"]=expected_fin.Enddt.dt.year
|
||||||
|
expected_gov=governance.reset_index()[["stkcd","Reptdt"]]
|
||||||
|
expected_gov["year"]=expected_gov.Reptdt.str[:4].astype(int)
|
||||||
|
expected_merge=expected_fin.merge(expected_gov,on=["stkcd","year"],how="outer",validate="1:1",indicator=True)
|
||||||
|
status_map={"left_only":1,"right_only":2,"both":3}
|
||||||
|
expected_merge["merge_status"]=expected_merge._merge.astype(str).map(status_map)
|
||||||
|
expected_merge=expected_merge.set_index(["stkcd","year"]).sort_index()
|
||||||
|
actual_merge=merged.set_index(["stkcd","year"]).sort_index()
|
||||||
|
check("merge_keyset",expected_merge.index.equals(actual_merge.index))
|
||||||
|
check("merge_status_every_row",np.array_equal(expected_merge.merge_status.to_numpy(),actual_merge._merge.to_numpy()))
|
||||||
|
check("in_resset_flag",np.array_equal(merged.in_resset.to_numpy(),merged._merge.isin([1,3]).astype(int).to_numpy()))
|
||||||
|
check("in_csmar_flag",np.array_equal(merged.in_csmar.to_numpy(),merged._merge.isin([2,3]).astype(int).to_numpy()))
|
||||||
|
check("matched_flag",np.array_equal(merged.matched.to_numpy(),(merged._merge==3).astype(int).to_numpy()))
|
||||||
|
check("complete_roa_roe_flag",np.array_equal(merged.complete_roa_roe.to_numpy(),((merged._merge==3)&merged.ROA.notna()&merged.ROE.notna()).astype(int).to_numpy()))
|
||||||
|
check("unusual_code_retained",merged.loc[merged.code_format_exception==1,["stkcd","year"]].to_dict(orient="records")==[{"stkcd":"0991","year":1998},{"stkcd":"0991","year":1999}])
|
||||||
|
|
||||||
|
expected_annual=expected_merge.reset_index().groupby(["year","merge_status"]).size().unstack(fill_value=0)
|
||||||
|
actual_annual=merged.groupby(["year","_merge"]).size().unstack(fill_value=0)
|
||||||
|
check("annual_merge_counts",expected_annual.equals(actual_annual))
|
||||||
|
|
||||||
|
def verify_csv_against_dta(stem):
|
||||||
|
dta=pd.read_stata(ROOT/(stem+".dta"),convert_categoricals=False)
|
||||||
|
numeric=[col for col in dta if pd.api.types.is_numeric_dtype(dta[col].dtype)]
|
||||||
|
dates=[col for col in dta if pd.api.types.is_datetime64_any_dtype(dta[col].dtype)]
|
||||||
|
types={col:(float if col in numeric else str) for col in dta}
|
||||||
|
csv=pd.read_csv(ROOT/(stem+".csv"),dtype=types,keep_default_na=False,
|
||||||
|
na_values={col:[""] for col in numeric},float_precision="round_trip",encoding="utf-8")
|
||||||
|
check(stem+" CSV column_names_order",list(dta.columns)==list(csv.columns),column_count=len(csv.columns))
|
||||||
|
for col in dates:
|
||||||
|
date_strings=csv[col]
|
||||||
|
valid=date_strings.eq("") | date_strings.str.fullmatch(r"\d{4}-\d{2}-\d{2}")
|
||||||
|
check(stem+" CSV ISO date "+col,valid.all())
|
||||||
|
csv[col]=pd.to_datetime(date_strings.mask(date_strings.eq("")),format="%Y-%m-%d",errors="raise")
|
||||||
|
compare_frames(stem+" DTA_vs_CSV",dta,csv,require_exact_numeric=True)
|
||||||
|
return {"rows":len(csv),"columns":len(csv.columns),"numeric_columns":len(numeric),
|
||||||
|
"date_columns":dates,"cells_compared":int(csv.size),"numeric_rule":"17-digit CSV parsed with float_precision=round_trip; exact equality including missing values"}
|
||||||
|
|
||||||
|
success_marker="WEEK8_ANALYSIS_COMPLETED_SUCCESSFULLY"
|
||||||
|
check("Stata_success_log",success_marker in (ROOT/"week8_stata.log").read_text(encoding="utf-8"))
|
||||||
|
results["csv_validation"]={stem:verify_csv_against_dta(stem) for stem in ["financial_governance_all_years","df_example_metrics"]}
|
||||||
|
results["annual_counts"]={str(y):{str(k):int(v) for k,v in row.items()} for y,row in actual_annual.to_dict(orient="index").items()}
|
||||||
|
results["summary"]={"financial_rows":len(financial),"governance_rows":len(governance),"merged_rows":len(merged),"merge_counts":{str(k):int(v) for k,v in merged._merge.value_counts().sort_index().items()},"financial_cells_compared_per_dataset":int(financial.size),"governance_cells_compared_per_dataset":int(governance.size),"matched_complete_roa_roe":int(merged.complete_roa_roe.sum()),"matched_missing_roa":int(merged.loc[merged.matched==1,"ROA"].isna().sum()),"matched_missing_roe":int(merged.loc[merged.matched==1,"ROE"].isna().sum())}
|
||||||
|
results["source_sha256"]=source_hashes
|
||||||
|
results["check_count"]=len(results["checks"])
|
||||||
|
results["summary"]["all_cells_compared"]=sum(x.get("cells_compared",0) for x in results["checks"])
|
||||||
|
(ROOT/"validation_results.json").write_text(json.dumps(results,ensure_ascii=False,indent=2,allow_nan=False),encoding="utf-8")
|
||||||
|
print(json.dumps({"passed":results["passed"],"check_count":results["check_count"],"summary":results["summary"]},ensure_ascii=False,indent=2))
|
||||||
|
if not results["passed"]:
|
||||||
|
raise SystemExit(1)
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
variable,description,source_unit,source
|
||||||
|
"stkcd","证券代码文本;通常六位;原始0991的两条记录保留原样","文本标识","derived"
|
||||||
|
"code_format_exception","源代码非六位标记;0991两条记录未推测补零","0/1","derived"
|
||||||
|
"year","报告截止日期的年份,不是公告年份","年","derived"
|
||||||
|
"ROA","资产净利率;保留原值,不乘除100,经济尺度需原库数据字典确认","源表标注%;原值","RESSET"
|
||||||
|
"ROE","净资产收益率(摊薄);保留原值,不乘除100,经济尺度待核","源表标注%;原值","RESSET"
|
||||||
|
"Comcd","上市公司代码","文本标识","RESSET"
|
||||||
|
"Lcomnm","最新公司全称,不能直接视为当年公司名称","文本","RESSET"
|
||||||
|
"A_Stkcd","A股股票代码;原始列","文本标识","RESSET"
|
||||||
|
"B_Stkcd","B股股票代码;本文件全空","文本标识","RESSET"
|
||||||
|
"H_Stkcd","H股股票代码;本文件全空","文本标识","RESSET"
|
||||||
|
"Lstflg","源文件上市标识;全为A,不额外推断历史上市状态","原始编码","RESSET"
|
||||||
|
"Enddt","RESSET报告截止日期","日期","RESSET"
|
||||||
|
"Reporttype","报表类型;本文件全为Q4且日期均为12月31日","原始编码","RESSET"
|
||||||
|
"resset_excel_row","RESSET原始工作表行号","行","provenance"
|
||||||
|
"Reptdt","统计截止日期","没有单位","CSMAR"
|
||||||
|
"Annodt","公告日期","没有单位","CSMAR"
|
||||||
|
"Y0301b","股本结构是否变化","没有单位","CSMAR"
|
||||||
|
"Y0401b","股东总数","个","CSMAR"
|
||||||
|
"Y0501b","前十大股东是否存在关联","没有单位","CSMAR"
|
||||||
|
"Y0601b","员工人数","个","CSMAR"
|
||||||
|
"Y0701b","离退休职工人数","个","CSMAR"
|
||||||
|
"Y0801b","董事长名称","没有单位","CSMAR"
|
||||||
|
"Y0901b","总经理名称","没有单位","CSMAR"
|
||||||
|
"Y1001b","董事长与总经理兼任情况","没有单位","CSMAR"
|
||||||
|
"Y1101a","董事人数","个","CSMAR"
|
||||||
|
"Y1101b","其中:独立董事人数","个","CSMAR"
|
||||||
|
"Y1201a","监事总规模","个","CSMAR"
|
||||||
|
"Y1301b","高管人数","个","CSMAR"
|
||||||
|
"Bddihldn","董事会持股数量","股","CSMAR"
|
||||||
|
"Bsuphldn","监事会持股数量","股","CSMAR"
|
||||||
|
"Excuhldn","高管持股数量","股","CSMAR"
|
||||||
|
"Mngmhldn","管理层持股数量","股","CSMAR"
|
||||||
|
"Y1401b","年薪披露方式","没有单位","CSMAR"
|
||||||
|
"Y1501a","董事、监事及高管年薪总额","元","CSMAR"
|
||||||
|
"Y1501b","其中:董事、监事及高管津贴总额","元","CSMAR"
|
||||||
|
"Y1501c","董事、监事及高管前三名薪酬总额","元","CSMAR"
|
||||||
|
"Y1501d","董事前三名薪酬总额","元","CSMAR"
|
||||||
|
"Y1501e","高管前三名薪酬总额","元","CSMAR"
|
||||||
|
"Y1601a","未领取薪酬的董事、监事及高管人数","个","CSMAR"
|
||||||
|
"Y1601b","其中:未领取薪酬董事人数","个","CSMAR"
|
||||||
|
"Y1601c","其中:未领取薪酬监事人数","个","CSMAR"
|
||||||
|
"Y1701a","委员会设立总数","个","CSMAR"
|
||||||
|
"Y1701b","其中:四委设立个数","个","CSMAR"
|
||||||
|
"Y1701c","其中:其他委员会设立个数","个","CSMAR"
|
||||||
|
"Y1901b","备注","没有单位","CSMAR"
|
||||||
|
"Y1801b","独立董事与上市公司工作地点一致性统计","没有单位","CSMAR"
|
||||||
|
"csmar_date","由Reptdt解析的统计截止日期","日期","derived"
|
||||||
|
"csmar_announcement","由Annodt解析的公告日期;不用于当期年份匹配","日期","derived"
|
||||||
|
"csmar_source","原CSMAR文件名","文本","provenance"
|
||||||
|
"csmar_excel_row","CSMAR原始工作表行号","行","provenance"
|
||||||
|
"_merge","合并状态:1仅RESSET,2仅CSMAR,3两库匹配","编码1/2/3","derived"
|
||||||
|
"in_resset","是否在RESSET文件出现;1是0否","0/1","derived"
|
||||||
|
"in_csmar","是否在CSMAR文件出现;1是0否","0/1","derived"
|
||||||
|
"matched","是否两库匹配;1是0否","0/1","derived"
|
||||||
|
"complete_roa_roe","两库匹配且ROA/ROE均非缺失;1是0否","0/1","derived"
|
||||||
|
@@ -0,0 +1,313 @@
|
|||||||
|
version 15.0
|
||||||
|
clear all
|
||||||
|
set more off
|
||||||
|
set type double
|
||||||
|
set linesize 160
|
||||||
|
capture log close _all
|
||||||
|
* Edit only this working-directory line if the folder is moved.
|
||||||
|
cd "C:/Users/cattom/Documents/econometrics/Week8"
|
||||||
|
log using "week8_stata.log", text replace
|
||||||
|
display "Week 8: residual degrees of freedom and RESSET-CSMAR merge"
|
||||||
|
display "All financial values are preserved in their original source scale."
|
||||||
|
|
||||||
|
tempname audit dict
|
||||||
|
postfile `audit' str60 metric double value using "processing_audit.dta", replace
|
||||||
|
postfile `dict' str32 variable str140 description str32 source_unit str18 source using "variable_dictionary.dta", replace
|
||||||
|
|
||||||
|
* 1. RESSET: read data below its Chinese header, retaining numeric precision.
|
||||||
|
import excel using "data/FININDRATIO_OLD_5938AD12016_(锐思).xls", sheet("FININDRATIO_OLD") cellrange(A2) clear
|
||||||
|
rename (A B C D E F G H I J) (Comcd Lcomnm A_Stkcd B_Stkcd H_Stkcd Lstflg Enddt Reporttype ROA ROE)
|
||||||
|
post `audit' ("resset_input_rows") (_N)
|
||||||
|
generate long resset_excel_row = _n + 1
|
||||||
|
generate str6 stkcd = strtrim(A_Stkcd)
|
||||||
|
assert regexm(stkcd,"^[0-9]+$") & strlen(stkcd) <= 6
|
||||||
|
generate byte code_format_exception = strlen(stkcd) != 6
|
||||||
|
assert stkcd == "0991" & inlist(year(Enddt),1998,1999) if code_format_exception
|
||||||
|
quietly count if code_format_exception
|
||||||
|
post `audit' ("resset_code_format_exceptions") (r(N))
|
||||||
|
assert Lstflg == "A"
|
||||||
|
assert missing(B_Stkcd) & missing(H_Stkcd)
|
||||||
|
assert Reporttype == "Q4"
|
||||||
|
assert month(Enddt) == 12 & day(Enddt) == 31
|
||||||
|
generate int year = year(Enddt)
|
||||||
|
assert !missing(stkcd,year)
|
||||||
|
isid stkcd year
|
||||||
|
post `audit' ("resset_duplicate_key_excess") (0)
|
||||||
|
quietly summarize year
|
||||||
|
post `audit' ("resset_min_year") (r(min))
|
||||||
|
post `audit' ("resset_max_year") (r(max))
|
||||||
|
egen byte firm_tag = tag(stkcd)
|
||||||
|
quietly count if firm_tag
|
||||||
|
post `audit' ("resset_distinct_codes") (r(N))
|
||||||
|
drop firm_tag
|
||||||
|
foreach v in ROA ROE {
|
||||||
|
quietly count if missing(`v')
|
||||||
|
post `audit' ("resset_missing_`v'") (r(N))
|
||||||
|
}
|
||||||
|
format Enddt %tdCCYY-NN-DD
|
||||||
|
format ROA ROE %24.17g
|
||||||
|
label variable stkcd "Source security code as text; normally six digits"
|
||||||
|
label variable code_format_exception "Source code has nonstandard length; preserved without guessing"
|
||||||
|
label variable year "Report year, not announcement year"
|
||||||
|
label variable ROA "资产净利率;原表标注(%);原值未换算"
|
||||||
|
label variable ROE "净资产收益率(摊薄);原表标注(%);原值未换算"
|
||||||
|
label variable Enddt "RESSET报告截止日期"
|
||||||
|
label variable resset_excel_row "Row number in original RESSET worksheet"
|
||||||
|
label variable Lcomnm "源文件中的最新公司全称,非逐年历史名称"
|
||||||
|
post `dict' ("stkcd") ("证券代码文本;通常六位;原始0991的两条记录保留原样") ("文本标识") ("derived")
|
||||||
|
post `dict' ("code_format_exception") ("源代码非六位标记;0991两条记录未推测补零") ("0/1") ("derived")
|
||||||
|
post `dict' ("year") ("报告截止日期的年份,不是公告年份") ("年") ("derived")
|
||||||
|
post `dict' ("ROA") ("资产净利率;保留原值,不乘除100,经济尺度需原库数据字典确认") ("源表标注%;原值") ("RESSET")
|
||||||
|
post `dict' ("ROE") ("净资产收益率(摊薄);保留原值,不乘除100,经济尺度待核") ("源表标注%;原值") ("RESSET")
|
||||||
|
post `dict' ("Comcd") ("上市公司代码") ("文本标识") ("RESSET")
|
||||||
|
post `dict' ("Lcomnm") ("最新公司全称,不能直接视为当年公司名称") ("文本") ("RESSET")
|
||||||
|
post `dict' ("A_Stkcd") ("A股股票代码;原始列") ("文本标识") ("RESSET")
|
||||||
|
post `dict' ("B_Stkcd") ("B股股票代码;本文件全空") ("文本标识") ("RESSET")
|
||||||
|
post `dict' ("H_Stkcd") ("H股股票代码;本文件全空") ("文本标识") ("RESSET")
|
||||||
|
post `dict' ("Lstflg") ("源文件上市标识;全为A,不额外推断历史上市状态") ("原始编码") ("RESSET")
|
||||||
|
post `dict' ("Enddt") ("RESSET报告截止日期") ("日期") ("RESSET")
|
||||||
|
post `dict' ("Reporttype") ("报表类型;本文件全为Q4且日期均为12月31日") ("原始编码") ("RESSET")
|
||||||
|
post `dict' ("resset_excel_row") ("RESSET原始工作表行号") ("行") ("provenance")
|
||||||
|
compress
|
||||||
|
sort stkcd year
|
||||||
|
save "resset_financial_all_years.dta", replace
|
||||||
|
|
||||||
|
* 2. CSMAR: retain every original governance field; append the two periods.
|
||||||
|
tempfile gov_first
|
||||||
|
foreach period in "1999-2012" "2013-2014" {
|
||||||
|
import excel using "data/CG_Ybasic`period'.xls", sheet("CG_Ybasic") firstrow allstring clear
|
||||||
|
assert Stkcd[1] == "证券代码"
|
||||||
|
assert Stkcd[2] == "没有单位"
|
||||||
|
foreach v of varlist _all {
|
||||||
|
local description = `v'[1]
|
||||||
|
local unit = `v'[2]
|
||||||
|
label variable `v' "`description'"
|
||||||
|
if "`period'" == "1999-2012" & "`v'" != "Stkcd" {
|
||||||
|
post `dict' ("`v'") ("`description'") ("`unit'") ("CSMAR")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
drop in 1/2
|
||||||
|
generate long csmar_excel_row = _n + 3
|
||||||
|
post `audit' ("csmar_`period'_input_rows") (_N)
|
||||||
|
rename Stkcd stkcd
|
||||||
|
replace stkcd = strtrim(stkcd)
|
||||||
|
assert regexm(stkcd,"^[0-9][0-9][0-9][0-9][0-9][0-9]$")
|
||||||
|
generate double csmar_date = daily(Reptdt,"YMD")
|
||||||
|
generate double csmar_announcement = daily(Annodt,"YMD")
|
||||||
|
assert !missing(csmar_date)
|
||||||
|
assert !missing(csmar_announcement) if strtrim(Annodt) != ""
|
||||||
|
assert month(csmar_date) == 12 & day(csmar_date) == 31
|
||||||
|
generate int year = year(csmar_date)
|
||||||
|
assert inrange(year,1999,2012) if "`period'" == "1999-2012"
|
||||||
|
assert inrange(year,2013,2014) if "`period'" == "2013-2014"
|
||||||
|
destring Y0301b Y0401b Y0501b Y0601b Y0701b Y1001b Y1101a Y1101b Y1201a Y1301b Bddihldn Bsuphldn Excuhldn Mngmhldn Y1401b Y1501a Y1501b Y1501c Y1501d Y1501e Y1601a Y1601b Y1601c Y1701a Y1701b Y1701c Y1801b, replace
|
||||||
|
generate str28 csmar_source = "CG_Ybasic`period'.xls"
|
||||||
|
format csmar_date csmar_announcement %tdCCYY-NN-DD
|
||||||
|
isid stkcd year
|
||||||
|
if "`period'" == "1999-2012" save `gov_first', replace
|
||||||
|
}
|
||||||
|
append using `gov_first'
|
||||||
|
isid stkcd year
|
||||||
|
post `audit' ("csmar_total_rows") (_N)
|
||||||
|
post `audit' ("csmar_duplicate_key_excess") (0)
|
||||||
|
egen byte firm_tag = tag(stkcd)
|
||||||
|
quietly count if firm_tag
|
||||||
|
post `audit' ("csmar_distinct_codes") (r(N))
|
||||||
|
drop firm_tag
|
||||||
|
label variable csmar_date "CSMAR统计截止日期;由Reptdt解析"
|
||||||
|
label variable csmar_announcement "CSMAR公告日期;由Annodt解析"
|
||||||
|
label variable csmar_source "Original CSMAR filename"
|
||||||
|
label variable csmar_excel_row "Row number in original CSMAR worksheet"
|
||||||
|
post `dict' ("csmar_date") ("由Reptdt解析的统计截止日期") ("日期") ("derived")
|
||||||
|
post `dict' ("csmar_announcement") ("由Annodt解析的公告日期;不用于当期年份匹配") ("日期") ("derived")
|
||||||
|
post `dict' ("csmar_source") ("原CSMAR文件名") ("文本") ("provenance")
|
||||||
|
post `dict' ("csmar_excel_row") ("CSMAR原始工作表行号") ("行") ("provenance")
|
||||||
|
sort stkcd year
|
||||||
|
compress
|
||||||
|
save "csmar_governance_1999_2014.dta", replace
|
||||||
|
|
||||||
|
* 3. Full outer 1:1 merge: retain all years and explicitly mark unmatched rows.
|
||||||
|
use "resset_financial_all_years.dta", clear
|
||||||
|
merge 1:1 stkcd year using "csmar_governance_1999_2014.dta"
|
||||||
|
assert _merge == 1 if code_format_exception == 1
|
||||||
|
replace code_format_exception = 0 if _merge == 2
|
||||||
|
assert Enddt == csmar_date if _merge == 3
|
||||||
|
isid stkcd year
|
||||||
|
generate byte in_resset = inlist(_merge,1,3)
|
||||||
|
generate byte in_csmar = inlist(_merge,2,3)
|
||||||
|
generate byte matched = _merge == 3
|
||||||
|
generate byte complete_roa_roe = matched & !missing(ROA,ROE)
|
||||||
|
label define merge_source 1 "RESSET only" 2 "CSMAR only" 3 "Matched", replace
|
||||||
|
label values _merge merge_source
|
||||||
|
label variable complete_roa_roe "Matched record with both ROA and ROE observed"
|
||||||
|
foreach v in in_resset in_csmar matched complete_roa_roe {
|
||||||
|
label variable `v' "`v': 1=yes, 0=no"
|
||||||
|
}
|
||||||
|
post `dict' ("_merge") ("合并状态:1仅RESSET,2仅CSMAR,3两库匹配") ("编码1/2/3") ("derived")
|
||||||
|
post `dict' ("in_resset") ("是否在RESSET文件出现;1是0否") ("0/1") ("derived")
|
||||||
|
post `dict' ("in_csmar") ("是否在CSMAR文件出现;1是0否") ("0/1") ("derived")
|
||||||
|
post `dict' ("matched") ("是否两库匹配;1是0否") ("0/1") ("derived")
|
||||||
|
post `dict' ("complete_roa_roe") ("两库匹配且ROA/ROE均非缺失;1是0否") ("0/1") ("derived")
|
||||||
|
post `audit' ("merged_total_rows") (_N)
|
||||||
|
forvalues s = 1/3 {
|
||||||
|
quietly count if _merge == `s'
|
||||||
|
post `audit' ("merge_status_`s'") (r(N))
|
||||||
|
}
|
||||||
|
quietly count if complete_roa_roe
|
||||||
|
post `audit' ("matched_complete_roa_roe") (r(N))
|
||||||
|
egen byte firm_tag = tag(stkcd) if matched
|
||||||
|
quietly count if firm_tag == 1
|
||||||
|
post `audit' ("matched_distinct_codes") (r(N))
|
||||||
|
drop firm_tag
|
||||||
|
quietly count if _merge == 1 & year < 1999
|
||||||
|
post `audit' ("resset_only_before_1999") (r(N))
|
||||||
|
quietly count if _merge == 1 & year >= 1999
|
||||||
|
post `audit' ("resset_only_from_1999") (r(N))
|
||||||
|
foreach v in ROA ROE {
|
||||||
|
quietly count if matched & missing(`v')
|
||||||
|
post `audit' ("matched_missing_`v'") (r(N))
|
||||||
|
}
|
||||||
|
order stkcd year ROA ROE _merge in_resset in_csmar matched complete_roa_roe
|
||||||
|
sort stkcd year
|
||||||
|
format ROA ROE %24.17g
|
||||||
|
char _dta[scope] "All observations supplied by user; not a certified census of listed firms."
|
||||||
|
char _dta[units] "ROA/ROE raw source values retained; headers say percent; economic scale not independently verified."
|
||||||
|
char _dta[merge] "1:1 stkcd year; financial master, governance using; full outer merge."
|
||||||
|
save "financial_governance_all_years.dta", replace
|
||||||
|
preserve
|
||||||
|
* Stata display formats can round large holdings/pay values in CSV.
|
||||||
|
* Export all non-date numeric columns with full double precision.
|
||||||
|
quietly ds, has(type numeric)
|
||||||
|
local numeric_vars `r(varlist)'
|
||||||
|
local date_vars Enddt csmar_date csmar_announcement
|
||||||
|
local precise_vars : list numeric_vars - date_vars
|
||||||
|
format `precise_vars' %24.17g
|
||||||
|
export delimited using "financial_governance_all_years.csv", datafmt nolabel quote replace
|
||||||
|
restore
|
||||||
|
preserve
|
||||||
|
keep if matched
|
||||||
|
save "financial_governance_matched.dta", replace
|
||||||
|
restore
|
||||||
|
preserve
|
||||||
|
keep if !matched
|
||||||
|
keep stkcd year _merge ROA ROE Comcd Lcomnm Enddt Reptdt csmar_source resset_excel_row csmar_excel_row
|
||||||
|
export delimited using "unmatched_records.csv", datafmt nolabel quote replace
|
||||||
|
restore
|
||||||
|
|
||||||
|
* 4. Year-by-year coverage and missingness; structural absence is separate.
|
||||||
|
preserve
|
||||||
|
generate byte resset_only = _merge == 1
|
||||||
|
generate byte csmar_only = _merge == 2
|
||||||
|
generate byte missing_roa = in_resset & missing(ROA)
|
||||||
|
generate byte missing_roe = in_resset & missing(ROE)
|
||||||
|
collapse (sum) resset_n=in_resset csmar_n=in_csmar matched_n=matched complete_n=complete_roa_roe resset_only csmar_only missing_roa missing_roe, by(year)
|
||||||
|
generate double pct_resset_matched = 100*matched_n/resset_n
|
||||||
|
generate double pct_csmar_matched = 100*matched_n/csmar_n
|
||||||
|
format pct* %12.6f
|
||||||
|
save "merge_by_year.dta", replace
|
||||||
|
export delimited using "merge_by_year.csv", datafmt replace
|
||||||
|
list, noobs separator(0)
|
||||||
|
set scheme s2color
|
||||||
|
twoway (line resset_n year, lcolor(navy) lwidth(medthick)) (line csmar_n year, lcolor(teal) lwidth(medthick)) (line matched_n year, lcolor(orange) lwidth(medthick)), title("Annual coverage in the supplied files") subtitle("Observations by report year; not a market census") xtitle("Report year") ytitle("Company-year records") xlabel(1990(2)2014, angle(45)) legend(order(1 "RESSET" 2 "CSMAR" 3 "Matched") rows(1)) graphregion(color(white)) name(coverage, replace)
|
||||||
|
graph save "annual_coverage.gph", replace
|
||||||
|
graph export "annual_coverage.png", width(1800) replace
|
||||||
|
restore
|
||||||
|
|
||||||
|
* 5. Descriptive statistics on the unchanged source scale.
|
||||||
|
tempname desc
|
||||||
|
postfile `desc' str20 sample str8 variable long observations long missing_count double mean double sd double min double p1 double p25 double p50 double p75 double p99 double max using "descriptive_statistics.dta", replace
|
||||||
|
foreach sample in resset_all matched matched_complete {
|
||||||
|
local condition "in_resset == 1"
|
||||||
|
if "`sample'" == "matched" local condition "matched == 1"
|
||||||
|
if "`sample'" == "matched_complete" local condition "complete_roa_roe == 1"
|
||||||
|
foreach v in ROA ROE {
|
||||||
|
quietly count if (`condition') & missing(`v')
|
||||||
|
local nm = r(N)
|
||||||
|
quietly summarize `v' if `condition', detail
|
||||||
|
post `desc' ("`sample'") ("`v'") (r(N)) (`nm') (r(mean)) (r(sd)) (r(min)) (r(p1)) (r(p25)) (r(p50)) (r(p75)) (r(p99)) (r(max))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
postclose `desc'
|
||||||
|
preserve
|
||||||
|
use "descriptive_statistics.dta", clear
|
||||||
|
format mean-max %24.17g
|
||||||
|
export delimited using "descriptive_statistics.csv", datafmt replace
|
||||||
|
list sample variable observations missing_count mean p50 min max, noobs
|
||||||
|
restore
|
||||||
|
|
||||||
|
* Preserve a short list of source extremes for transparent follow-up.
|
||||||
|
preserve
|
||||||
|
keep if in_resset
|
||||||
|
keep stkcd year Comcd Lcomnm ROA ROE resset_excel_row
|
||||||
|
generate byte extreme = 0
|
||||||
|
foreach v in ROA ROE {
|
||||||
|
quietly summarize `v', detail
|
||||||
|
replace extreme = 1 if inlist(`v',r(min),r(max)) & !missing(`v')
|
||||||
|
}
|
||||||
|
keep if extreme
|
||||||
|
drop extreme
|
||||||
|
export delimited using "source_extremes.csv", datafmt quote replace
|
||||||
|
restore
|
||||||
|
|
||||||
|
* 6. Ordinary OLS example to verify residual df; no causal interpretation.
|
||||||
|
* Using 2013 (a single cross-section) avoids implying panel-error independence.
|
||||||
|
generate double board_size = Y1101a
|
||||||
|
generate double independent_share = Y1101b/Y1101a if Y1101a > 0 & !missing(Y1101a,Y1101b)
|
||||||
|
generate byte valid_board = board_size > 0 & !missing(board_size,independent_share) & inrange(independent_share,0,1)
|
||||||
|
quietly count if matched & year == 2013 & !valid_board
|
||||||
|
post `audit' ("ols_2013_excluded_board_records") (r(N))
|
||||||
|
regress ROA board_size independent_share if matched & year == 2013 & valid_board
|
||||||
|
estimates save "df_example_ols.ster", replace
|
||||||
|
scalar n_ols = e(N)
|
||||||
|
scalar k_ols = e(df_m)
|
||||||
|
scalar rank_ols = e(rank)
|
||||||
|
scalar df_ols = e(df_r)
|
||||||
|
scalar sse_ols = e(rss)
|
||||||
|
scalar s2_ols = e(rss)/e(df_r)
|
||||||
|
scalar rmse2_ols = e(rmse)^2
|
||||||
|
scalar naive_s2 = e(rss)/e(N)
|
||||||
|
assert scalar(k_ols) == 2
|
||||||
|
assert scalar(rank_ols) == 3
|
||||||
|
assert scalar(df_ols) == scalar(n_ols)-3
|
||||||
|
assert reldif(scalar(s2_ols),scalar(rmse2_ols)) < 1e-12
|
||||||
|
predict double residual if e(sample), residuals
|
||||||
|
generate double residual_sq = residual^2
|
||||||
|
quietly summarize residual_sq, meanonly
|
||||||
|
scalar residual_sse = r(sum)
|
||||||
|
assert reldif(scalar(residual_sse),scalar(sse_ols)) < 1e-10
|
||||||
|
quietly summarize residual, meanonly
|
||||||
|
scalar residual_sum = r(sum)
|
||||||
|
generate double board_residual = board_size*residual
|
||||||
|
quietly summarize board_residual, meanonly
|
||||||
|
scalar board_residual_sum = r(sum)
|
||||||
|
generate double independent_residual = independent_share*residual
|
||||||
|
quietly summarize independent_residual, meanonly
|
||||||
|
scalar independent_residual_sum = r(sum)
|
||||||
|
assert abs(scalar(residual_sum)) < 1e-7
|
||||||
|
assert abs(scalar(board_residual_sum)) < 1e-6
|
||||||
|
assert abs(scalar(independent_residual_sum)) < 1e-7
|
||||||
|
tempname dfout
|
||||||
|
postfile `dfout' str36 metric double value using "df_example_metrics.dta", replace
|
||||||
|
foreach s in n_ols k_ols rank_ols df_ols sse_ols s2_ols rmse2_ols naive_s2 residual_sse residual_sum board_residual_sum independent_residual_sum {
|
||||||
|
post `dfout' ("`s'") (scalar(`s'))
|
||||||
|
}
|
||||||
|
postclose `dfout'
|
||||||
|
preserve
|
||||||
|
keep if e(sample)
|
||||||
|
keep stkcd year ROA board_size independent_share residual residual_sq
|
||||||
|
save "df_example_sample.dta", replace
|
||||||
|
restore
|
||||||
|
postclose `audit'
|
||||||
|
postclose `dict'
|
||||||
|
|
||||||
|
foreach f in processing_audit variable_dictionary df_example_metrics {
|
||||||
|
use "`f'.dta", clear
|
||||||
|
capture format value %24.17g
|
||||||
|
export delimited using "`f'.csv", datafmt quote replace
|
||||||
|
}
|
||||||
|
use "financial_governance_all_years.dta", clear
|
||||||
|
isid stkcd year
|
||||||
|
tabulate _merge
|
||||||
|
display "WEEK8_ANALYSIS_COMPLETED_SUCCESSFULLY"
|
||||||
|
log close
|
||||||
|
exit, clear
|
||||||