Add documentation for the Clang.Token module. #23

src/Clang.hs

@@ -81,8 +81,7 @@ module Clang (
 , FFI.SourceRange
 , FFI.ClangString
 , FFI.Token
-, FFI.TokenKind
-, FFI.TokenList
+, FFI.TokenKind(..)
 , FFI.Index
 , FFI.ReparseFlags
 , FFI.SaveTranslationUnitFlags(..)

src/Clang/Internal/FFI.gc

@@ -2945,7 +2945,8 @@ type_getOffsetOf t field = liftIO $ B.useAsCString field $ \cField ->
 %     CXType r = clang_getIBOutletCollectionType(a);
 %result (type {r.kind} {r.data[0]} {r.data[1]})
 
--- We deliberately don't export the constructor. The only way to unwrap this is registerCursorList.
+-- We deliberately don't export the constructor for UnsafeCursorList.
+-- The only way to unwrap it is registerCursorList.
 type CursorList s = DVS.Vector (Cursor s)
 instance ClangValueList Cursor
 data UnsafeCursorList = UnsafeCursorList !(Ptr ()) !Int
@@ -3061,7 +3062,8 @@ instance Storable (ParentedCursor s) where
       pokeByteOff p offsetParentedCursorCursor child
     {-# INLINE poke #-}
 
--- We deliberately don't export the constructor. The only way to unwrap this is registerParentedCursorList.
+-- We deliberately don't export the constructor for UnsafeParentedCursorList.
+-- The only way to unwrap it is registerParentedCursorList.
 type ParentedCursorList s = DVS.Vector (ParentedCursor s)
 instance ClangValueList ParentedCursor
 data UnsafeParentedCursorList = UnsafeParentedCursorList !(Ptr ()) !Int
@@ -3909,7 +3911,13 @@ fullComment_getAsXML = registerClangString . unsafe_FullComment_getAsXML
 --   CXToken_Literal,
 --   CXToken_Comment
 -- } CXTokenKind;
-%enum TokenKind (Bounded, Enum, Eq, Ord, Read, Show) Int [CXToken_Punctuation, CXToken_Keyword, CXToken_Identifier, CXToken_Literal, CXToken_Comment]
+%enum TokenKind (Bounded, Enum, Eq, Ord, Read, Show) Int
+%  [ PunctuationToken = CXToken_Punctuation
+%  , KeywordToken     = CXToken_Keyword
+%  , IdentifierToken  = CXToken_Identifier
+%  , LiteralToken     = CXToken_Literal
+%  , CommentToken     = CXToken_Comment
+%  ]
 
 -- typedef struct {
 --   unsigned int_data[4];
@@ -3977,7 +3985,8 @@ getTokenSpelling = (registerClangString .) . unsafe_getTokenSpelling
 %     CXSourceRange r = clang_getTokenExtent(t, a);
 %result (sourceRange {r.ptr_data[0]} {r.ptr_data[1]} {r.begin_int_data} {r.end_int_data})
 
--- We deliberately don't export the constructor. The only way to unwrap this is registerCursorList.
+-- We deliberately don't export the constructor for UnsafeTokenList.
+-- The only way to unwrap it is registerTokenList.
 type TokenList s = DVS.Vector (Token s)
 instance ClangValueList Token
 data UnsafeTokenList = UnsafeTokenList !(Ptr ()) !Int

src/Clang/Token.hs

@@ -1,45 +1,90 @@
 {-# LANGUAGE ConstraintKinds #-}
 {-# LANGUAGE FlexibleContexts #-}
 
+-- | Functions for manipulating 'Clang.Token's.
+--
+-- This module is intended to be imported qualified.
 module Clang.Token
 (
--- * Getting information about tokens
-  getKind
+-- * Token lists
+  tokenize
+, List
+
+-- * Token kinds
+, getKind
+
+-- * Mapping between tokens and source code
 , getSpelling
 , getLocation
 , getExtent
-
--- * Converting to and from lists of tokens
-, tokenize
 , annotateTokens
 ) where
 
 import Control.Monad.IO.Class
+import qualified Data.Vector.Storable as DVS
 
 import qualified Clang.Internal.FFI as FFI
 import Clang.Internal.Monad
 
+-- | Tokenizes the source code described by the given range into raw
+-- lexical tokens.
+tokenize :: ClangBase m
+         => FFI.TranslationUnit s'  -- ^ The translation list
+                                    -- containing the source code.
+         -> FFI.SourceRange s''     -- ^ The source range in which text
+                                    -- should be tokenized.
+         -> ClangT s m (List s)
+tokenize = FFI.tokenize
+
+-- | A list of tokens, stored as a 'DVS.Vector' for efficiency.
+type List s = DVS.Vector (FFI.Token s)
+
+-- | Determines the kind of the given token, expressed as a
+-- 'FFI.TokenKind' value.
 getKind :: ClangBase m => FFI.Token s' -> ClangT s m FFI.TokenKind
 getKind t = liftIO $ FFI.getTokenKind t
 
+-- | Retrieves the \'spelling\', or textual representation, of the
+-- given token.
 getSpelling :: ClangBase m => FFI.TranslationUnit s' -> FFI.Token s'' -> ClangT s m (FFI.ClangString s)
 getSpelling = FFI.getTokenSpelling
 
+-- | Returns the source location of the given token.
 getLocation :: ClangBase m => FFI.TranslationUnit s' -> FFI.Token s''
             -> ClangT s m (FFI.SourceLocation s)
 getLocation tu tk = liftIO $ FFI.getTokenLocation mkProxy tu tk
 
+-- | Retrieves the source range that covers the given token.
 getExtent :: ClangBase m => FFI.TranslationUnit s' -> FFI.Token s''
           -> ClangT s m (FFI.SourceRange s)
 getExtent tu tk = liftIO $ FFI.getTokenExtent mkProxy tu tk
 
-tokenize :: ClangBase m => FFI.TranslationUnit s' -> FFI.SourceRange s''
-         -> ClangT s m (FFI.TokenList s)
-tokenize = FFI.tokenize
-
-annotateTokens ::
-     ClangBase m =>
-     FFI.TranslationUnit s' -- ^ The translation unit related to the tokens
-  -> FFI.TokenList s'' -- ^ Token list that you want cursors for
-  -> ClangT s m (FFI.CursorList s) -- ^ Cursors corresponding to the tokens
+-- | Annotates the given set of tokens by providing cursors for each token
+-- that can be mapped to a specific entity within the abstract syntax tree.
+-- 
+-- This is equivalent to invoking 'Clang.Cursor.getCursor' on the
+-- source locations of each of these tokens. The cursors provided are
+-- filtered, so that only those cursors that have a direct
+-- correspondence to the token are accepted. For example, given a
+-- function call \'f(x)\', 'Clang.Cursor.getCursor' would provide the
+-- following cursors:
+-- 
+--   * When the cursor is over the \'f\', a 'Clang.DeclRefExpr' cursor
+--     referring to \'f\'.
+--
+--   * When the cursor is over the \'(\' or the \')\', a
+--     'Clang.CallExpr' referring to \'f\'.
+--
+--   * When the cursor is over the \'x\', a 'Clang.DeclRefExpr' cursor
+--     referring to \'x\'.
+-- 
+-- Only the first and last of these cursors will occur within the
+-- annotate, since the tokens \'f\' and \'x\' directly refer to a function
+-- and a variable, respectively, but the parentheses are just a small
+-- part of the full syntax of the function call expression, which is
+-- not provided as an annotation.
+annotateTokens :: ClangBase m
+               => FFI.TranslationUnit s' -- ^ The translation unit that owns the tokens.
+               -> List s''               -- ^ The tokens to annotate.
+               -> ClangT s m (FFI.CursorList s)
 annotateTokens = FFI.annotateTokens